{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from tqdm import tqdm_notebook as tqdm\n",
    "import preprocessor as p\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%run ../twitter15/twitter15_text_processing.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torchvision.transforms as transforms\n",
    "import torchvision.datasets as dsets\n",
    "from torch.autograd import Variable\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pickle as pkl\n",
    "from collections import defaultdict\n",
    "import pandas as pd\n",
    "import os\n",
    "import numpy as np\n",
    "import json\n",
    "from tqdm import tqdm, tqdm_notebook\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import classification_report, f1_score, accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "import spacy\n",
    "from tqdm import tqdm, tqdm_notebook, tnrange\n",
    "import pandas as pd\n",
    "from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score, classification_report, confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Node:\n",
    "    def __init__(self,uid,tid,time_stamp,label):\n",
    "        self.children = {}\n",
    "        self.childrenList = []\n",
    "        self.num_children = 0\n",
    "        self.tid = tid\n",
    "        self.uid = uid\n",
    "        self.label = label\n",
    "        self.time_stamp = time_stamp\n",
    "    \n",
    "    def add_child(self,node):\n",
    "        if node.uid not in self.children:\n",
    "            self.children[node.uid] = node\n",
    "            self.num_children += 1\n",
    "        else:\n",
    "            self.children[node.uid] = node\n",
    "        self.childrenList = list(self.children.values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Tree:\n",
    "    def __init__(self,root):\n",
    "        self.root = root\n",
    "        self.tweet_id = root.tid\n",
    "        self.uid = root.uid\n",
    "        self.height = 0\n",
    "        self.nodes = 0\n",
    "    \n",
    "    def show(self):\n",
    "        queue = [self.root,0]\n",
    "        \n",
    "        while len(queue) != 0:\n",
    "            toprint = queue.pop(0)\n",
    "            if toprint == 0:\n",
    "                print('\\n')\n",
    "            else:\n",
    "                print(toprint.uid,end=' ')\n",
    "                queue += toprint.children.values()\n",
    "                queue.append(0)\n",
    "                \n",
    "    def insertnode(self,curnode,parent,child):\n",
    "        if curnode.uid == parent.uid:\n",
    "            curnode.add_child(child)\n",
    "            return 1\n",
    "\n",
    "        elif parent.uid in curnode.children:\n",
    "            s = self.insertnode(curnode.children[parent.uid],parent,child)\n",
    "            return 2\n",
    "        else:\n",
    "            for node in curnode.children:\n",
    "                s = self.insertnode(curnode.children[node],parent,child)\n",
    "                if s == 2:\n",
    "                    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def loadPklFileNum(datapath,incSize,fileNum):\n",
    "    \n",
    "    with open(datapath+str(incSize)+'inc_'+str(fileNum)+'.pickle', 'rb') as handle:\n",
    "        twitTrees = pkl.load(handle)\n",
    "    return twitTrees"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def loadTreeFilesOfIncrement(datapath,incSize):\n",
    "    twittertrees = {}\n",
    "    \n",
    "    files = [x for x in os.listdir(t15Datapath) if str(incSize)+'inc' in x]\n",
    "    \n",
    "    for file in tqdm(files):\n",
    "        with open(datapath+file,'rb') as handle:\n",
    "            partialTrees = pkl.load(handle)\n",
    "        twittertrees.update(partialTrees)\n",
    "        \n",
    "    return twittertrees"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "t15Datapath = '../twitter15/pickledTrees/'\n",
    "# twitter15_trees = loadPklFileNum(t15Datapath,20,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 9/9 [01:22<00:00,  9.19s/it]\n"
     ]
    }
   ],
   "source": [
    "twitter15_trees = loadTreeFilesOfIncrement(t15Datapath,20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|██████████| 1000/1000 [00:00<00:00, 25946.98it/s]\n",
      "\n",
      "  0%|          | 0/33 [00:00<?, ?it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n",
      "30\n",
      "31\n",
      "32\n",
      "33\n",
      "34\n",
      "35\n",
      "36\n",
      "37\n",
      "38\n",
      "39\n",
      "40\n",
      "41\n",
      "42\n",
      "43\n",
      "44\n",
      "45\n",
      "46\n",
      "47\n",
      "48\n",
      "49\n",
      "50\n",
      "51\n",
      "52\n",
      "53\n",
      "54\n",
      "55\n",
      "56\n",
      "57\n",
      "58\n",
      "59\n",
      "60\n",
      "61\n",
      "62\n",
      "63\n",
      "64\n",
      "65\n",
      "66\n",
      "67\n",
      "68\n",
      "69\n",
      "70\n",
      "71\n",
      "72\n",
      "73\n",
      "74\n",
      "75\n",
      "76\n",
      "77\n",
      "78\n",
      "79\n",
      "80\n",
      "81\n",
      "82\n",
      "83\n",
      "84\n",
      "85\n",
      "86\n",
      "87\n",
      "88\n",
      "89\n",
      "90\n",
      "91\n",
      "92\n",
      "93\n",
      "94\n",
      "95\n",
      "96\n",
      "97\n",
      "98\n",
      "99\n",
      "100\n",
      "101\n",
      "102\n",
      "103\n",
      "104\n",
      "105\n",
      "106\n",
      "107\n",
      "108\n",
      "109\n",
      "110\n",
      "111\n",
      "112\n",
      "113\n",
      "114\n",
      "115\n",
      "116\n",
      "117\n",
      "118\n",
      "119\n",
      "120\n",
      "121\n",
      "122\n",
      "123\n",
      "124\n",
      "125\n",
      "126\n",
      "127\n",
      "128\n",
      "129\n",
      "130\n",
      "131\n",
      "132\n",
      "133\n",
      "134\n",
      "135\n",
      "136\n",
      "137\n",
      "138\n",
      "139\n",
      "140\n",
      "141\n",
      "142\n",
      "143\n",
      "144\n",
      "145\n",
      "146\n",
      "147\n",
      "148\n",
      "149\n",
      "150\n",
      "151\n",
      "152\n",
      "153\n",
      "154\n",
      "155\n",
      "156\n",
      "157\n",
      "158\n",
      "159\n",
      "160\n",
      "161\n",
      "162\n",
      "163\n",
      "164\n",
      "165\n",
      "166\n",
      "167\n",
      "168\n",
      "169\n",
      "170\n",
      "171\n",
      "172\n",
      "173\n",
      "174\n",
      "175\n",
      "176\n",
      "177\n",
      "178\n",
      "179\n",
      "180\n",
      "181\n",
      "182\n",
      "183\n",
      "184\n",
      "185\n",
      "186\n",
      "187\n",
      "188\n",
      "189\n",
      "190\n",
      "191\n",
      "192\n",
      "193\n",
      "194\n",
      "195\n",
      "196\n",
      "197\n",
      "198\n",
      "199\n",
      "200\n",
      "201\n",
      "202\n",
      "203\n",
      "204\n",
      "205\n",
      "206\n",
      "207\n",
      "208\n",
      "209\n",
      "210\n",
      "211\n",
      "212\n",
      "213\n",
      "214\n",
      "215\n",
      "216\n",
      "217\n",
      "218\n",
      "219\n",
      "220\n",
      "221\n",
      "222\n",
      "223\n",
      "224\n",
      "225\n",
      "226\n",
      "227\n",
      "228\n",
      "229\n",
      "230\n",
      "231\n",
      "232\n",
      "233\n",
      "234\n",
      "235\n",
      "236\n",
      "237\n",
      "238\n",
      "239\n",
      "240\n",
      "241\n",
      "242\n",
      "243\n",
      "244\n",
      "245\n",
      "246\n",
      "247\n",
      "248\n",
      "249\n",
      "250\n",
      "251\n",
      "252\n",
      "253\n",
      "254\n",
      "255\n",
      "256\n",
      "257\n",
      "258\n",
      "259\n",
      "260\n",
      "261\n",
      "262\n",
      "263\n",
      "264\n",
      "265\n",
      "266\n",
      "267\n",
      "268\n",
      "269\n",
      "270\n",
      "271\n",
      "272\n",
      "273\n",
      "274\n",
      "275\n",
      "276\n",
      "277\n",
      "278\n",
      "279\n",
      "280\n",
      "281\n",
      "282\n",
      "283\n",
      "284\n",
      "285\n",
      "286\n",
      "287\n",
      "288\n",
      "289\n",
      "290\n",
      "291\n",
      "292\n",
      "293\n",
      "294\n",
      "295\n",
      "296\n",
      "297\n",
      "298\n",
      "299\n",
      "300\n",
      "301\n",
      "302\n",
      "303\n",
      "304\n",
      "305\n",
      "306\n",
      "307\n",
      "308\n",
      "309\n",
      "310\n",
      "311\n",
      "312\n",
      "313\n",
      "314\n",
      "315\n",
      "316\n",
      "317\n",
      "318\n",
      "319\n",
      "320\n",
      "321\n",
      "322\n",
      "323\n",
      "324\n",
      "325\n",
      "326\n",
      "327\n",
      "328\n",
      "329\n",
      "330\n",
      "331\n",
      "332\n",
      "333\n",
      "334\n",
      "335\n",
      "336\n",
      "337\n",
      "338\n",
      "339\n",
      "340\n",
      "341\n",
      "342\n",
      "343\n",
      "344\n",
      "345\n",
      "346\n",
      "347\n",
      "348\n",
      "349\n",
      "350\n",
      "351\n",
      "352\n",
      "353\n",
      "354\n",
      "355\n",
      "356\n",
      "357\n",
      "358\n",
      "359\n",
      "360\n",
      "361\n",
      "362\n",
      "363\n",
      "364\n",
      "365\n",
      "366\n",
      "367\n",
      "368\n",
      "369\n",
      "370\n",
      "371\n",
      "372\n",
      "373\n",
      "374\n",
      "375\n",
      "376\n",
      "377\n",
      "378\n",
      "379\n",
      "380\n",
      "381\n",
      "382\n",
      "383\n",
      "384\n",
      "385\n",
      "386\n",
      "387\n",
      "388\n",
      "389\n",
      "390\n",
      "391\n",
      "392\n",
      "393\n",
      "394\n",
      "395\n",
      "396\n",
      "397\n",
      "398\n",
      "399\n",
      "400\n",
      "401\n",
      "402\n",
      "403\n",
      "404\n",
      "405\n",
      "406\n",
      "407\n",
      "408\n",
      "409\n",
      "410\n",
      "411\n",
      "412\n",
      "413\n",
      "414\n",
      "415\n",
      "416\n",
      "417\n",
      "418\n",
      "419\n",
      "420\n",
      "421\n",
      "422\n",
      "423\n",
      "424\n",
      "425\n",
      "426\n",
      "427\n",
      "428\n",
      "429\n",
      "430\n",
      "431\n",
      "432\n",
      "433\n",
      "434\n",
      "435\n",
      "436\n",
      "437\n",
      "438\n",
      "439\n",
      "440\n",
      "441\n",
      "442\n",
      "443\n",
      "444\n",
      "445\n",
      "446\n",
      "447\n",
      "448\n",
      "449\n",
      "450\n",
      "451\n",
      "452\n",
      "453\n",
      "454\n",
      "455\n",
      "456\n",
      "457\n",
      "458\n",
      "459\n",
      "460\n",
      "461\n",
      "462\n",
      "463\n",
      "464\n",
      "465\n",
      "466\n",
      "467\n",
      "468\n",
      "469\n",
      "470\n",
      "471\n",
      "472\n",
      "473\n",
      "474\n",
      "475\n",
      "476\n",
      "477\n",
      "478\n",
      "479\n",
      "480\n",
      "481\n",
      "482\n",
      "483\n",
      "484\n",
      "485\n",
      "486\n",
      "487\n",
      "488\n",
      "489\n",
      "490\n",
      "491\n",
      "492\n",
      "493\n",
      "494\n",
      "495\n",
      "496\n",
      "497\n",
      "498\n",
      "499\n",
      "500\n",
      "501\n",
      "502\n",
      "503\n",
      "504\n",
      "505\n",
      "506\n",
      "507\n",
      "508\n",
      "509\n",
      "510\n",
      "511\n",
      "512\n",
      "513\n",
      "514\n",
      "515\n",
      "516\n",
      "517\n",
      "518\n",
      "519\n",
      "520\n",
      "521\n",
      "522\n",
      "523\n",
      "524\n",
      "525\n",
      "526\n",
      "527\n",
      "528\n",
      "529\n",
      "530\n",
      "531\n",
      "532\n",
      "533\n",
      "534\n",
      "535\n",
      "536\n",
      "537\n",
      "538\n",
      "539\n",
      "540\n",
      "541\n",
      "542\n",
      "543\n",
      "544\n",
      "545\n",
      "546\n",
      "547\n",
      "548\n",
      "549\n",
      "550\n",
      "551\n",
      "552\n",
      "553\n",
      "554\n",
      "555\n",
      "556\n",
      "557\n",
      "558\n",
      "559\n",
      "560\n",
      "561\n",
      "562\n",
      "563\n",
      "564\n",
      "565\n",
      "566\n",
      "567\n",
      "568\n",
      "569\n",
      "570\n",
      "571\n",
      "572\n",
      "573\n",
      "574\n",
      "575\n",
      "576\n",
      "577\n",
      "578\n",
      "579\n",
      "580\n",
      "581\n",
      "582\n",
      "583\n",
      "584\n",
      "585\n",
      "586\n",
      "587\n",
      "588\n",
      "589\n",
      "590\n",
      "591\n",
      "592\n",
      "593\n",
      "594\n",
      "595\n",
      "596\n",
      "597\n",
      "598\n",
      "599\n",
      "600\n",
      "601\n",
      "602\n",
      "603\n",
      "604\n",
      "605\n",
      "606\n",
      "607\n",
      "608\n",
      "609\n",
      "610\n",
      "611\n",
      "612\n",
      "613\n",
      "614\n",
      "615\n",
      "616\n",
      "617\n",
      "618\n",
      "619\n",
      "620\n",
      "621\n",
      "622\n",
      "623\n",
      "624\n",
      "625\n",
      "626\n",
      "627\n",
      "628\n",
      "629\n",
      "630\n",
      "631\n",
      "632\n",
      "633\n",
      "634\n",
      "635\n",
      "636\n",
      "637\n",
      "638\n",
      "639\n",
      "640\n",
      "641\n",
      "642\n",
      "643\n",
      "644\n",
      "645\n",
      "646\n",
      "647\n",
      "648\n",
      "649\n",
      "650\n",
      "651\n",
      "652\n",
      "653\n",
      "654\n",
      "655\n",
      "656\n",
      "657\n",
      "658\n",
      "659\n",
      "660\n",
      "661\n",
      "662\n",
      "663\n",
      "664\n",
      "665\n",
      "666\n",
      "667\n",
      "668\n",
      "669\n",
      "670\n",
      "671\n",
      "672\n",
      "673\n",
      "674\n",
      "675\n",
      "676\n",
      "677\n",
      "678\n",
      "679\n",
      "680\n",
      "681\n",
      "682\n",
      "683\n",
      "684\n",
      "685\n",
      "686\n",
      "687\n",
      "688\n",
      "689\n",
      "690\n",
      "691\n",
      "692\n",
      "693\n",
      "694\n",
      "695\n",
      "696\n",
      "697\n",
      "698\n",
      "699\n",
      "700\n",
      "701\n",
      "702\n",
      "703\n",
      "704\n",
      "705\n",
      "706\n",
      "707\n",
      "708\n",
      "709\n",
      "710\n",
      "711\n",
      "712\n",
      "713\n",
      "714\n",
      "715\n",
      "716\n",
      "717\n",
      "718\n",
      "719\n",
      "720\n",
      "721\n",
      "722\n",
      "723\n",
      "724\n",
      "725\n",
      "726\n",
      "727\n",
      "728\n",
      "729\n",
      "730\n",
      "731\n",
      "732\n",
      "733\n",
      "734\n",
      "735\n",
      "736\n",
      "737\n",
      "738\n",
      "739\n",
      "740\n",
      "741\n",
      "742\n",
      "743\n",
      "744\n",
      "745\n",
      "746\n",
      "747\n",
      "748\n",
      "749\n",
      "750\n",
      "751\n",
      "752\n",
      "753\n",
      "754\n",
      "755\n",
      "756\n",
      "757\n",
      "758\n",
      "759\n",
      "760\n",
      "761\n",
      "762\n",
      "763\n",
      "764\n",
      "765\n",
      "766\n",
      "767\n",
      "768\n",
      "769\n",
      "770\n",
      "771\n",
      "772\n",
      "773\n",
      "774\n",
      "775\n",
      "776\n",
      "777\n",
      "778\n",
      "779\n",
      "780\n",
      "781\n",
      "782\n",
      "783\n",
      "784\n",
      "785\n",
      "786\n",
      "787\n",
      "788\n",
      "789\n",
      "790\n",
      "791\n",
      "792\n",
      "793\n",
      "794\n",
      "795\n",
      "796\n",
      "797\n",
      "798\n",
      "799\n",
      "800\n",
      "801\n",
      "802\n",
      "803\n",
      "804\n",
      "805\n",
      "806\n",
      "807\n",
      "808\n",
      "809\n",
      "810\n",
      "811\n",
      "812\n",
      "813\n",
      "814\n",
      "815\n",
      "816\n",
      "817\n",
      "818\n",
      "819\n",
      "820\n",
      "821\n",
      "822\n",
      "823\n",
      "824\n",
      "825\n",
      "826\n",
      "827\n",
      "828\n",
      "829\n",
      "830\n",
      "831\n",
      "832\n",
      "833\n",
      "834\n",
      "835\n",
      "836\n",
      "837\n",
      "838\n",
      "839\n",
      "840\n",
      "841\n",
      "842\n",
      "843\n",
      "844\n",
      "845\n",
      "846\n",
      "847\n",
      "848\n",
      "849\n",
      "850\n",
      "851\n",
      "852\n",
      "853\n",
      "854\n",
      "855\n",
      "856\n",
      "857\n",
      "858\n",
      "859\n",
      "860\n",
      "861\n",
      "862\n",
      "863\n",
      "864\n",
      "865\n",
      "866\n",
      "867\n",
      "868\n",
      "869\n",
      "870\n",
      "871\n",
      "872\n",
      "873\n",
      "874\n",
      "875\n",
      "876\n",
      "877\n",
      "878\n",
      "879\n",
      "880\n",
      "881\n",
      "882\n",
      "883\n",
      "884\n",
      "885\n",
      "886\n",
      "887\n",
      "888\n",
      "889\n",
      "890\n",
      "891\n",
      "892\n",
      "893\n",
      "894\n",
      "895\n",
      "896\n",
      "897\n",
      "898\n",
      "899\n",
      "900\n",
      "901\n",
      "902\n",
      "903\n",
      "904\n",
      "905\n",
      "906\n",
      "907\n",
      "908\n",
      "909\n",
      "910\n",
      "911\n",
      "912\n",
      "913\n",
      "914\n",
      "915\n",
      "916\n",
      "917\n",
      "918\n",
      "919\n",
      "920\n",
      "921\n",
      "922\n",
      "923\n",
      "924\n",
      "925\n",
      "926\n",
      "927\n",
      "928\n",
      "929\n",
      "930\n",
      "931\n",
      "932\n",
      "933\n",
      "934\n",
      "935\n",
      "936\n",
      "937\n",
      "938\n",
      "939\n",
      "940\n",
      "941\n",
      "942\n",
      "943\n",
      "944\n",
      "945\n",
      "946\n",
      "947\n",
      "948\n",
      "949\n",
      "950\n",
      "951\n",
      "952\n",
      "953\n",
      "954\n",
      "955\n",
      "956\n",
      "957\n",
      "958\n",
      "959\n",
      "960\n",
      "961\n",
      "962\n",
      "963\n",
      "964\n",
      "965\n",
      "966\n",
      "967\n",
      "968\n",
      "969\n",
      "970\n",
      "971\n",
      "972\n",
      "973\n",
      "974\n",
      "975\n",
      "976\n",
      "977\n",
      "978\n",
      "979\n",
      "980\n",
      "981\n",
      "982\n",
      "983\n",
      "984\n",
      "985\n",
      "986\n",
      "987\n",
      "988\n",
      "989\n",
      "990\n",
      "991\n",
      "992\n",
      "993\n",
      "994\n",
      "995\n",
      "996\n",
      "997\n",
      "998\n",
      "999\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "  3%|▎         | 1/33 [00:03<01:43,  3.23s/it]\u001b[A\n",
      "  6%|▌         | 2/33 [00:06<01:38,  3.19s/it]\u001b[A\n",
      "  9%|▉         | 3/33 [00:09<01:35,  3.17s/it]\u001b[A\n",
      " 12%|█▏        | 4/33 [00:15<01:56,  4.00s/it]\u001b[A\n",
      " 15%|█▌        | 5/33 [00:18<01:44,  3.74s/it]\u001b[A\n",
      " 18%|█▊        | 6/33 [00:22<01:41,  3.75s/it]\u001b[A\n",
      " 21%|██        | 7/33 [00:26<01:41,  3.90s/it]\u001b[A\n",
      " 24%|██▍       | 8/33 [00:29<01:31,  3.68s/it]\u001b[A\n",
      " 27%|██▋       | 9/33 [00:32<01:24,  3.50s/it]\u001b[A\n",
      " 30%|███       | 10/33 [00:35<01:18,  3.40s/it]\u001b[A\n",
      " 33%|███▎      | 11/33 [00:44<01:47,  4.90s/it]\u001b[A\n",
      " 36%|███▋      | 12/33 [00:47<01:31,  4.34s/it]\u001b[A\n",
      " 39%|███▉      | 13/33 [00:50<01:19,  4.00s/it]\u001b[A\n",
      " 42%|████▏     | 14/33 [00:53<01:11,  3.75s/it]\u001b[A\n",
      " 45%|████▌     | 15/33 [00:56<01:03,  3.55s/it]\u001b[A\n",
      " 48%|████▊     | 16/33 [00:59<00:58,  3.42s/it]\u001b[A\n",
      " 52%|█████▏    | 17/33 [01:03<00:52,  3.31s/it]\u001b[A\n",
      " 55%|█████▍    | 18/33 [01:07<00:53,  3.54s/it]\u001b[A\n",
      " 58%|█████▊    | 19/33 [01:14<01:04,  4.63s/it]\u001b[A\n",
      " 61%|██████    | 20/33 [01:18<00:58,  4.51s/it]\u001b[A\n",
      " 64%|██████▎   | 21/33 [01:22<00:52,  4.41s/it]\u001b[A\n",
      " 67%|██████▋   | 22/33 [01:26<00:45,  4.12s/it]\u001b[A\n",
      " 70%|██████▉   | 23/33 [01:29<00:38,  3.88s/it]\u001b[A\n",
      " 73%|███████▎  | 24/33 [01:32<00:33,  3.68s/it]\u001b[A\n",
      " 76%|███████▌  | 25/33 [01:36<00:29,  3.71s/it]\u001b[A\n",
      " 79%|███████▉  | 26/33 [01:39<00:24,  3.51s/it]\u001b[A\n",
      " 82%|████████▏ | 27/33 [01:42<00:20,  3.48s/it]\u001b[A\n",
      " 85%|████████▍ | 28/33 [01:46<00:16,  3.38s/it]\u001b[A\n",
      " 88%|████████▊ | 29/33 [01:53<00:18,  4.70s/it]\u001b[A\n",
      " 91%|█████████ | 30/33 [01:56<00:12,  4.17s/it]\u001b[A\n",
      " 94%|█████████▍| 31/33 [01:59<00:07,  3.87s/it]\u001b[A\n",
      " 97%|█████████▋| 32/33 [02:03<00:03,  3.81s/it]\u001b[A\n",
      "100%|██████████| 33/33 [02:06<00:00,  3.84s/it]\u001b[A\n",
      "\n",
      "  0%|          | 0/253378 [00:00<?, ?it/s]\u001b[A\n",
      "  0%|          | 7/253378 [00:00<1:00:23, 69.92it/s]\u001b[A\n",
      "  0%|          | 420/253378 [00:00<42:30, 99.17it/s]\u001b[A\n",
      "  0%|          | 835/253378 [00:00<30:00, 140.23it/s]\u001b[A\n",
      "  0%|          | 1251/253378 [00:00<21:16, 197.47it/s]\u001b[A\n",
      "  1%|          | 1664/253378 [00:00<15:10, 276.44it/s]\u001b[A\n",
      "  1%|          | 2082/253378 [00:00<10:54, 384.02it/s]\u001b[A\n",
      "  1%|          | 2497/253378 [00:00<07:55, 527.64it/s]\u001b[A\n",
      "  1%|          | 2913/253378 [00:00<05:50, 714.87it/s]\u001b[A\n",
      "  1%|▏         | 3329/253378 [00:00<04:22, 951.11it/s]\u001b[A\n",
      "  1%|▏         | 3745/253378 [00:01<03:21, 1237.47it/s]\u001b[A\n",
      "  2%|▏         | 4159/253378 [00:01<02:39, 1566.84it/s]\u001b[A\n",
      "  2%|▏         | 4574/253378 [00:01<02:09, 1926.01it/s]\u001b[A\n",
      "  2%|▏         | 4989/253378 [00:01<01:48, 2294.53it/s]\u001b[A\n",
      "  2%|▏         | 5403/253378 [00:01<01:33, 2648.42it/s]\u001b[A\n",
      "  2%|▏         | 5814/253378 [00:01<01:23, 2963.93it/s]\u001b[A\n",
      "  2%|▏         | 6231/253378 [00:01<01:16, 3245.37it/s]\u001b[A\n",
      "  3%|▎         | 6645/253378 [00:01<01:11, 3469.54it/s]\u001b[A\n",
      "  3%|▎         | 7058/253378 [00:01<01:08, 3619.66it/s]\u001b[A\n",
      "  3%|▎         | 7474/253378 [00:01<01:05, 3765.74it/s]\u001b[A\n",
      "  3%|▎         | 7891/253378 [00:02<01:03, 3878.52it/s]\u001b[A\n",
      "  3%|▎         | 8305/253378 [00:02<01:02, 3951.93it/s]\u001b[A\n",
      "  3%|▎         | 8719/253378 [00:02<01:01, 3987.64it/s]\u001b[A\n",
      "  4%|▎         | 9136/253378 [00:02<01:00, 4039.00it/s]\u001b[A\n",
      "  4%|▍         | 9552/253378 [00:02<00:59, 4074.03it/s]\u001b[A\n",
      "  4%|▍         | 9971/253378 [00:02<00:59, 4105.44it/s]\u001b[A\n",
      "  4%|▍         | 10387/253378 [00:02<00:59, 4111.18it/s]\u001b[A\n",
      "  4%|▍         | 10804/253378 [00:02<00:58, 4126.80it/s]\u001b[A\n",
      "  4%|▍         | 11219/253378 [00:02<00:58, 4122.30it/s]\u001b[A\n",
      "  5%|▍         | 11636/253378 [00:02<00:58, 4135.50it/s]\u001b[A\n",
      "  5%|▍         | 12051/253378 [00:03<00:58, 4107.38it/s]\u001b[A\n",
      "  5%|▍         | 12465/253378 [00:03<00:58, 4116.81it/s]\u001b[A\n",
      "  5%|▌         | 12880/253378 [00:03<00:58, 4125.13it/s]\u001b[A\n",
      "  5%|▌         | 13294/253378 [00:03<00:58, 4129.17it/s]\u001b[A\n",
      "  5%|▌         | 13708/253378 [00:03<00:58, 4124.78it/s]\u001b[A\n",
      "  6%|▌         | 14127/253378 [00:03<00:57, 4142.22it/s]\u001b[A\n",
      "  6%|▌         | 14543/253378 [00:03<00:57, 4145.44it/s]\u001b[A\n",
      "  6%|▌         | 14958/253378 [00:03<00:57, 4143.22it/s]\u001b[A\n",
      "  6%|▌         | 15373/253378 [00:03<00:57, 4143.92it/s]\u001b[A\n",
      "  6%|▌         | 15789/253378 [00:03<00:57, 4147.70it/s]\u001b[A\n",
      "  6%|▋         | 16206/253378 [00:04<00:57, 4152.19it/s]\u001b[A\n",
      "  7%|▋         | 16622/253378 [00:04<00:57, 4151.24it/s]\u001b[A\n",
      "  7%|▋         | 17038/253378 [00:04<00:57, 4118.52it/s]\u001b[A\n",
      "  7%|▋         | 17457/253378 [00:04<00:57, 4137.65it/s]\u001b[A\n",
      "  7%|▋         | 17873/253378 [00:04<00:56, 4142.33it/s]\u001b[A\n",
      "  7%|▋         | 18292/253378 [00:04<00:56, 4153.69it/s]\u001b[A\n",
      "  7%|▋         | 18708/253378 [00:04<00:56, 4149.79it/s]\u001b[A\n",
      "  8%|▊         | 19124/253378 [00:04<00:56, 4148.88it/s]\u001b[A\n",
      "  8%|▊         | 19541/253378 [00:04<00:56, 4153.93it/s]\u001b[A\n",
      "  8%|▊         | 19958/253378 [00:04<00:56, 4155.82it/s]\u001b[A\n",
      "  8%|▊         | 20374/253378 [00:05<00:56, 4156.41it/s]\u001b[A\n",
      "  8%|▊         | 20794/253378 [00:05<00:55, 4166.84it/s]\u001b[A\n",
      "  8%|▊         | 21213/253378 [00:05<00:55, 4172.05it/s]\u001b[A\n",
      "  9%|▊         | 21631/253378 [00:05<00:55, 4162.36it/s]\u001b[A\n",
      "  9%|▊         | 22048/253378 [00:05<00:56, 4078.72it/s]\u001b[A\n",
      "  9%|▉         | 22465/253378 [00:05<00:56, 4103.68it/s]\u001b[A\n",
      "  9%|▉         | 22882/253378 [00:05<00:55, 4122.82it/s]\u001b[A\n",
      "  9%|▉         | 23296/253378 [00:05<00:55, 4125.41it/s]\u001b[A\n",
      "  9%|▉         | 23716/253378 [00:05<00:55, 4144.87it/s]\u001b[A\n",
      " 10%|▉         | 24134/253378 [00:05<00:55, 4154.72it/s]\u001b[A\n",
      " 10%|▉         | 24550/253378 [00:06<00:55, 4153.69it/s]\u001b[A\n",
      " 10%|▉         | 24969/253378 [00:06<00:54, 4161.97it/s]\u001b[A\n",
      " 10%|█         | 25386/253378 [00:06<00:55, 4118.53it/s]\u001b[A\n",
      " 10%|█         | 25798/253378 [00:06<00:55, 4084.64it/s]\u001b[A\n",
      " 10%|█         | 26212/253378 [00:06<00:55, 4100.49it/s]\u001b[A\n",
      " 11%|█         | 26629/253378 [00:06<00:55, 4120.41it/s]\u001b[A\n",
      " 11%|█         | 27043/253378 [00:06<00:54, 4123.40it/s]\u001b[A\n",
      " 11%|█         | 27460/253378 [00:06<00:54, 4136.80it/s]\u001b[A\n",
      " 11%|█         | 27876/253378 [00:06<00:54, 4141.14it/s]\u001b[A\n",
      " 11%|█         | 28295/253378 [00:06<00:54, 4153.00it/s]\u001b[A\n",
      " 11%|█▏        | 28712/253378 [00:07<00:54, 4155.11it/s]\u001b[A\n",
      " 11%|█▏        | 29131/253378 [00:07<00:53, 4164.14it/s]\u001b[A\n",
      " 12%|█▏        | 29548/253378 [00:07<00:53, 4157.50it/s]\u001b[A\n",
      " 12%|█▏        | 29968/253378 [00:07<00:53, 4168.02it/s]\u001b[A\n",
      " 12%|█▏        | 30385/253378 [00:07<00:53, 4160.11it/s]\u001b[A\n",
      " 12%|█▏        | 30805/253378 [00:07<00:53, 4171.60it/s]\u001b[A\n",
      " 12%|█▏        | 31223/253378 [00:07<00:53, 4155.27it/s]\u001b[A\n",
      " 12%|█▏        | 31640/253378 [00:07<00:53, 4157.22it/s]\u001b[A\n",
      " 13%|█▎        | 32056/253378 [00:07<00:53, 4116.01it/s]\u001b[A\n",
      " 13%|█▎        | 32475/253378 [00:07<00:53, 4136.56it/s]\u001b[A\n",
      " 13%|█▎        | 32891/253378 [00:08<00:53, 4141.40it/s]\u001b[A\n",
      " 13%|█▎        | 33310/253378 [00:08<00:52, 4154.38it/s]\u001b[A\n",
      " 13%|█▎        | 33726/253378 [00:08<00:53, 4124.32it/s]\u001b[A\n",
      " 13%|█▎        | 34145/253378 [00:08<00:52, 4141.21it/s]\u001b[A\n",
      " 14%|█▎        | 34562/253378 [00:08<00:52, 4148.29it/s]\u001b[A\n",
      " 14%|█▍        | 34984/253378 [00:08<00:52, 4168.10it/s]\u001b[A\n",
      " 14%|█▍        | 35402/253378 [00:08<00:52, 4169.57it/s]\u001b[A\n",
      " 14%|█▍        | 35823/253378 [00:08<00:52, 4179.03it/s]\u001b[A\n",
      " 14%|█▍        | 36241/253378 [00:08<00:52, 4174.25it/s]\u001b[A\n",
      " 14%|█▍        | 36662/253378 [00:08<00:51, 4183.68it/s]\u001b[A\n",
      " 15%|█▍        | 37081/253378 [00:09<00:51, 4181.68it/s]\u001b[A\n",
      " 15%|█▍        | 37500/253378 [00:09<00:51, 4172.73it/s]\u001b[A\n",
      " 15%|█▍        | 37918/253378 [00:09<00:52, 4134.53it/s]\u001b[A\n",
      " 15%|█▌        | 38332/253378 [00:09<00:52, 4134.70it/s]\u001b[A\n",
      " 15%|█▌        | 38746/253378 [00:09<00:52, 4122.59it/s]\u001b[A\n",
      " 15%|█▌        | 39159/253378 [00:09<00:52, 4119.51it/s]\u001b[A\n",
      " 16%|█▌        | 39571/253378 [00:09<00:51, 4115.58it/s]\u001b[A\n",
      " 16%|█▌        | 39983/253378 [00:09<00:58, 3636.49it/s]\u001b[A\n",
      " 16%|█▌        | 40357/253378 [00:09<01:08, 3117.28it/s]\u001b[A\n",
      " 16%|█▌        | 40761/253378 [00:10<01:03, 3346.28it/s]\u001b[A\n",
      " 16%|█▋        | 41176/253378 [00:10<00:59, 3550.76it/s]\u001b[A\n",
      " 16%|█▋        | 41592/253378 [00:10<00:57, 3712.69it/s]\u001b[A\n",
      " 17%|█▋        | 42009/253378 [00:10<00:55, 3837.78it/s]\u001b[A\n",
      " 17%|█▋        | 42420/253378 [00:10<00:53, 3915.04it/s]\u001b[A\n",
      " 17%|█▋        | 42826/253378 [00:10<00:53, 3955.65it/s]\u001b[A\n",
      " 17%|█▋        | 43240/253378 [00:10<00:52, 4007.82it/s]\u001b[A\n",
      " 17%|█▋        | 43659/253378 [00:10<00:51, 4059.56it/s]\u001b[A\n",
      " 17%|█▋        | 44068/253378 [00:10<00:52, 3996.24it/s]\u001b[A\n",
      " 18%|█▊        | 44484/253378 [00:10<00:51, 4043.47it/s]\u001b[A\n",
      " 18%|█▊        | 44898/253378 [00:11<00:51, 4069.95it/s]\u001b[A\n",
      " 18%|█▊        | 45314/253378 [00:11<00:50, 4095.84it/s]\u001b[A\n",
      " 18%|█▊        | 45725/253378 [00:11<00:50, 4090.48it/s]\u001b[A\n",
      " 18%|█▊        | 46135/253378 [00:11<00:50, 4069.80it/s]\u001b[A\n",
      " 18%|█▊        | 46543/253378 [00:11<00:59, 3491.53it/s]\u001b[A\n",
      " 19%|█▊        | 46952/253378 [00:11<00:56, 3650.89it/s]\u001b[A\n",
      " 19%|█▊        | 47361/253378 [00:11<00:54, 3770.16it/s]\u001b[A\n",
      " 19%|█▉        | 47769/253378 [00:11<00:53, 3857.02it/s]\u001b[A\n",
      " 19%|█▉        | 48175/253378 [00:11<00:52, 3913.98it/s]\u001b[A\n",
      " 19%|█▉        | 48584/253378 [00:12<00:51, 3965.10it/s]\u001b[A\n",
      " 19%|█▉        | 48990/253378 [00:12<00:51, 3992.50it/s]\u001b[A\n",
      " 19%|█▉        | 49402/253378 [00:12<00:50, 4027.77it/s]\u001b[A\n",
      " 20%|█▉        | 49810/253378 [00:12<00:50, 4040.75it/s]\u001b[A\n",
      " 20%|█▉        | 50221/253378 [00:12<00:50, 4059.73it/s]\u001b[A\n",
      " 20%|█▉        | 50628/253378 [00:12<00:49, 4055.57it/s]\u001b[A\n",
      " 20%|██        | 51035/253378 [00:12<00:50, 4042.90it/s]\u001b[A\n",
      " 20%|██        | 51440/253378 [00:12<00:50, 4008.59it/s]\u001b[A\n",
      " 20%|██        | 51856/253378 [00:12<00:49, 4052.07it/s]\u001b[A\n",
      " 21%|██        | 52268/253378 [00:12<00:49, 4071.45it/s]\u001b[A\n",
      " 21%|██        | 52684/253378 [00:13<00:48, 4096.85it/s]\u001b[A\n",
      " 21%|██        | 53097/253378 [00:13<00:48, 4106.24it/s]\u001b[A\n",
      " 21%|██        | 53514/253378 [00:13<00:48, 4124.45it/s]\u001b[A\n",
      " 21%|██▏       | 53928/253378 [00:13<00:48, 4126.56it/s]\u001b[A\n",
      " 21%|██▏       | 54346/253378 [00:13<00:48, 4141.54it/s]\u001b[A\n",
      " 22%|██▏       | 54761/253378 [00:13<00:47, 4138.82it/s]\u001b[A\n",
      " 22%|██▏       | 55175/253378 [00:13<00:47, 4130.84it/s]\u001b[A\n",
      " 22%|██▏       | 55589/253378 [00:13<00:48, 4096.21it/s]\u001b[A\n",
      " 22%|██▏       | 56004/253378 [00:13<00:47, 4112.02it/s]\u001b[A\n",
      " 22%|██▏       | 56417/253378 [00:13<00:47, 4114.99it/s]\u001b[A\n",
      " 22%|██▏       | 56834/253378 [00:14<00:47, 4130.78it/s]\u001b[A\n",
      " 23%|██▎       | 57248/253378 [00:14<00:47, 4130.42it/s]\u001b[A\n",
      " 23%|██▎       | 57662/253378 [00:14<00:47, 4081.53it/s]\u001b[A\n",
      " 23%|██▎       | 58074/253378 [00:14<00:47, 4092.10it/s]\u001b[A\n",
      " 23%|██▎       | 58487/253378 [00:14<00:47, 4102.80it/s]\u001b[A\n",
      " 23%|██▎       | 58898/253378 [00:14<00:47, 4095.34it/s]\u001b[A\n",
      " 23%|██▎       | 59315/253378 [00:14<00:47, 4114.29it/s]\u001b[A\n",
      " 24%|██▎       | 59728/253378 [00:14<00:47, 4118.68it/s]\u001b[A\n",
      " 24%|██▎       | 60144/253378 [00:14<00:46, 4129.31it/s]\u001b[A\n",
      " 24%|██▍       | 60557/253378 [00:14<00:46, 4126.99it/s]\u001b[A\n",
      " 24%|██▍       | 60972/253378 [00:15<00:46, 4132.74it/s]\u001b[A\n",
      " 24%|██▍       | 61386/253378 [00:15<00:46, 4130.99it/s]\u001b[A\n",
      " 24%|██▍       | 61801/253378 [00:15<00:46, 4133.68it/s]\u001b[A\n",
      " 25%|██▍       | 62215/253378 [00:15<00:46, 4128.00it/s]\u001b[A\n",
      " 25%|██▍       | 62631/253378 [00:15<00:46, 4134.86it/s]\u001b[A\n",
      " 25%|██▍       | 63045/253378 [00:15<00:46, 4131.09it/s]\u001b[A\n",
      " 25%|██▌       | 63462/253378 [00:15<00:45, 4140.70it/s]\u001b[A\n",
      " 25%|██▌       | 63877/253378 [00:15<00:46, 4074.27it/s]\u001b[A\n",
      " 25%|██▌       | 64292/253378 [00:15<00:46, 4096.25it/s]\u001b[A\n",
      " 26%|██▌       | 64703/253378 [00:15<00:46, 4100.13it/s]\u001b[A\n",
      " 26%|██▌       | 65120/253378 [00:16<00:45, 4118.20it/s]\u001b[A\n",
      " 26%|██▌       | 65533/253378 [00:16<00:45, 4118.96it/s]\u001b[A\n",
      " 26%|██▌       | 65948/253378 [00:16<00:45, 4128.05it/s]\u001b[A\n",
      " 26%|██▌       | 66361/253378 [00:16<00:45, 4124.78it/s]\u001b[A\n",
      " 26%|██▋       | 66777/253378 [00:16<00:45, 4132.83it/s]\u001b[A\n",
      " 27%|██▋       | 67191/253378 [00:16<00:45, 4119.53it/s]\u001b[A\n",
      " 27%|██▋       | 67606/253378 [00:16<00:45, 4127.10it/s]\u001b[A\n",
      " 27%|██▋       | 68019/253378 [00:16<00:45, 4113.95it/s]\u001b[A\n",
      " 27%|██▋       | 68434/253378 [00:16<00:44, 4123.18it/s]\u001b[A\n",
      " 27%|██▋       | 68847/253378 [00:16<00:44, 4116.59it/s]\u001b[A\n",
      " 27%|██▋       | 69263/253378 [00:17<00:44, 4127.35it/s]\u001b[A\n",
      " 27%|██▋       | 69676/253378 [00:17<00:44, 4125.73it/s]\u001b[A\n",
      " 28%|██▊       | 70089/253378 [00:17<00:44, 4091.44it/s]\u001b[A\n",
      " 28%|██▊       | 70499/253378 [00:17<00:44, 4086.86it/s]\u001b[A\n",
      " 28%|██▊       | 70916/253378 [00:17<00:44, 4109.63it/s]\u001b[A\n",
      " 28%|██▊       | 71328/253378 [00:17<00:44, 4105.09it/s]\u001b[A\n",
      " 28%|██▊       | 71740/253378 [00:17<00:44, 4108.30it/s]\u001b[A\n",
      " 28%|██▊       | 72154/253378 [00:17<00:44, 4116.90it/s]\u001b[A\n",
      " 29%|██▊       | 72566/253378 [00:17<00:43, 4116.90it/s]\u001b[A\n",
      " 29%|██▉       | 72982/253378 [00:17<00:43, 4127.18it/s]\u001b[A\n",
      " 29%|██▉       | 73395/253378 [00:18<00:43, 4125.23it/s]\u001b[A\n",
      " 29%|██▉       | 73809/253378 [00:18<00:43, 4129.16it/s]\u001b[A\n",
      " 29%|██▉       | 74222/253378 [00:18<00:43, 4124.74it/s]\u001b[A\n",
      " 29%|██▉       | 74636/253378 [00:18<00:43, 4127.25it/s]\u001b[A\n",
      " 30%|██▉       | 75049/253378 [00:18<00:43, 4127.75it/s]\u001b[A\n",
      " 30%|██▉       | 75465/253378 [00:18<00:43, 4136.12it/s]\u001b[A\n",
      " 30%|██▉       | 75879/253378 [00:18<00:43, 4117.50it/s]\u001b[A\n",
      " 30%|███       | 76291/253378 [00:18<00:43, 4099.52it/s]\u001b[A\n",
      " 30%|███       | 76701/253378 [00:18<00:43, 4055.85it/s]\u001b[A\n",
      " 30%|███       | 77116/253378 [00:18<00:43, 4081.75it/s]\u001b[A\n",
      " 31%|███       | 77527/253378 [00:19<00:42, 4089.99it/s]\u001b[A\n",
      " 31%|███       | 77940/253378 [00:19<00:42, 4101.64it/s]\u001b[A\n",
      " 31%|███       | 78351/253378 [00:19<00:42, 4097.81it/s]\u001b[A\n",
      " 31%|███       | 78766/253378 [00:19<00:42, 4110.92it/s]\u001b[A\n",
      " 31%|███       | 79179/253378 [00:19<00:42, 4115.80it/s]\u001b[A\n",
      " 31%|███▏      | 79595/253378 [00:19<00:42, 4127.19it/s]\u001b[A\n",
      " 32%|███▏      | 80008/253378 [00:19<00:42, 4122.47it/s]\u001b[A\n",
      " 32%|███▏      | 80424/253378 [00:19<00:41, 4132.35it/s]\u001b[A\n",
      " 32%|███▏      | 80838/253378 [00:19<00:41, 4125.04it/s]\u001b[A\n",
      " 32%|███▏      | 81251/253378 [00:19<00:41, 4124.94it/s]\u001b[A\n",
      " 32%|███▏      | 81664/253378 [00:20<00:41, 4124.20it/s]\u001b[A\n",
      " 32%|███▏      | 82079/253378 [00:20<00:41, 4129.93it/s]\u001b[A\n",
      " 33%|███▎      | 82493/253378 [00:20<00:41, 4110.52it/s]\u001b[A\n",
      " 33%|███▎      | 82905/253378 [00:20<00:41, 4062.94it/s]\u001b[A\n",
      " 33%|███▎      | 83315/253378 [00:20<00:41, 4072.35it/s]\u001b[A\n",
      " 33%|███▎      | 83729/253378 [00:20<00:41, 4092.27it/s]\u001b[A\n",
      " 33%|███▎      | 84139/253378 [00:20<00:41, 4081.43it/s]\u001b[A\n",
      " 33%|███▎      | 84552/253378 [00:20<00:41, 4094.26it/s]\u001b[A\n",
      " 34%|███▎      | 84962/253378 [00:20<00:41, 4095.86it/s]\u001b[A\n",
      " 34%|███▎      | 85377/253378 [00:20<00:40, 4110.24it/s]\u001b[A\n",
      " 34%|███▍      | 85789/253378 [00:21<00:40, 4112.45it/s]\u001b[A\n",
      " 34%|███▍      | 86204/253378 [00:21<00:40, 4122.29it/s]\u001b[A\n",
      " 34%|███▍      | 86617/253378 [00:21<00:40, 4098.86it/s]\u001b[A\n",
      " 34%|███▍      | 87027/253378 [00:21<00:40, 4092.77it/s]\u001b[A\n",
      " 35%|███▍      | 87437/253378 [00:21<00:41, 4042.77it/s]\u001b[A\n",
      " 35%|███▍      | 87847/253378 [00:21<00:40, 4057.84it/s]\u001b[A\n",
      " 35%|███▍      | 88253/253378 [00:21<00:40, 4056.38it/s]\u001b[A\n",
      " 35%|███▍      | 88663/253378 [00:21<00:40, 4068.36it/s]\u001b[A\n",
      " 35%|███▌      | 89070/253378 [00:21<00:40, 4061.55it/s]\u001b[A\n",
      " 35%|███▌      | 89479/253378 [00:21<00:40, 4066.83it/s]\u001b[A\n",
      " 35%|███▌      | 89886/253378 [00:22<00:40, 4061.88it/s]\u001b[A\n",
      " 36%|███▌      | 90293/253378 [00:22<00:40, 4064.18it/s]\u001b[A\n",
      " 36%|███▌      | 90700/253378 [00:22<00:40, 4058.68it/s]\u001b[A\n",
      " 36%|███▌      | 91109/253378 [00:22<00:39, 4064.96it/s]\u001b[A\n",
      " 36%|███▌      | 91516/253378 [00:22<00:39, 4062.49it/s]\u001b[A\n",
      " 36%|███▋      | 91924/253378 [00:22<00:39, 4067.18it/s]\u001b[A\n",
      " 36%|███▋      | 92331/253378 [00:22<00:39, 4051.56it/s]\u001b[A\n",
      " 37%|███▋      | 92738/253378 [00:22<00:39, 4056.27it/s]\u001b[A\n",
      " 37%|███▋      | 93144/253378 [00:22<00:39, 4053.65it/s]\u001b[A\n",
      " 37%|███▋      | 93550/253378 [00:22<00:39, 4054.17it/s]\u001b[A\n",
      " 37%|███▋      | 93956/253378 [00:23<00:39, 4052.54it/s]\u001b[A\n",
      " 37%|███▋      | 94364/253378 [00:23<00:39, 4058.60it/s]\u001b[A\n",
      " 37%|███▋      | 94770/253378 [00:23<00:39, 4052.49it/s]\u001b[A\n",
      " 38%|███▊      | 95177/253378 [00:23<00:38, 4056.99it/s]\u001b[A\n",
      " 38%|███▊      | 95583/253378 [00:23<00:38, 4055.06it/s]\u001b[A\n",
      " 38%|███▊      | 95997/253378 [00:23<00:38, 4078.78it/s]\u001b[A\n",
      " 38%|███▊      | 96410/253378 [00:23<00:38, 4092.07it/s]\u001b[A\n",
      " 38%|███▊      | 96826/253378 [00:23<00:38, 4109.70it/s]\u001b[A\n",
      " 38%|███▊      | 97241/253378 [00:23<00:37, 4119.04it/s]\u001b[A\n",
      " 39%|███▊      | 97659/253378 [00:23<00:37, 4136.45it/s]\u001b[A\n",
      " 39%|███▊      | 98073/253378 [00:24<00:37, 4135.67it/s]\u001b[A\n",
      " 39%|███▉      | 98490/253378 [00:24<00:37, 4143.85it/s]\u001b[A\n",
      " 39%|███▉      | 98905/253378 [00:24<00:37, 4143.58it/s]\u001b[A\n",
      " 39%|███▉      | 99322/253378 [00:24<00:37, 4151.25it/s]\u001b[A\n",
      " 39%|███▉      | 99738/253378 [00:24<00:37, 4147.32it/s]\u001b[A\n",
      " 40%|███▉      | 100155/253378 [00:24<00:36, 4152.91it/s]\u001b[A\n",
      " 40%|███▉      | 100571/253378 [00:24<00:36, 4151.43it/s]\u001b[A\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 40%|███▉      | 100987/253378 [00:24<00:36, 4142.19it/s]\u001b[A\n",
      " 40%|████      | 101402/253378 [00:24<00:36, 4138.51it/s]\u001b[A\n",
      " 40%|████      | 101818/253378 [00:24<00:36, 4144.63it/s]\u001b[A\n",
      " 40%|████      | 102233/253378 [00:25<00:36, 4106.30it/s]\u001b[A\n",
      " 41%|████      | 102650/253378 [00:25<00:36, 4124.96it/s]\u001b[A\n",
      " 41%|████      | 103063/253378 [00:25<00:36, 4124.21it/s]\u001b[A\n",
      " 41%|████      | 103479/253378 [00:25<00:36, 4133.92it/s]\u001b[A\n",
      " 41%|████      | 103895/253378 [00:25<00:36, 4140.86it/s]\u001b[A\n",
      " 41%|████      | 104313/253378 [00:25<00:35, 4149.70it/s]\u001b[A\n",
      " 41%|████▏     | 104728/253378 [00:25<00:35, 4146.15it/s]\u001b[A\n",
      " 41%|████▏     | 105147/253378 [00:25<00:35, 4157.64it/s]\u001b[A\n",
      " 42%|████▏     | 105563/253378 [00:25<00:35, 4155.20it/s]\u001b[A\n",
      " 42%|████▏     | 105982/253378 [00:25<00:35, 4164.31it/s]\u001b[A\n",
      " 42%|████▏     | 106399/253378 [00:26<00:35, 4119.38it/s]\u001b[A\n",
      " 42%|████▏     | 106817/253378 [00:26<00:35, 4137.04it/s]\u001b[A\n",
      " 42%|████▏     | 107232/253378 [00:26<00:35, 4139.87it/s]\u001b[A\n",
      " 42%|████▏     | 107649/253378 [00:26<00:35, 4147.35it/s]\u001b[A\n",
      " 43%|████▎     | 108064/253378 [00:26<00:35, 4146.32it/s]\u001b[A\n",
      " 43%|████▎     | 108479/253378 [00:26<00:34, 4145.98it/s]\u001b[A\n",
      " 43%|████▎     | 108894/253378 [00:26<00:34, 4145.52it/s]\u001b[A\n",
      " 43%|████▎     | 109309/253378 [00:26<00:34, 4144.28it/s]\u001b[A\n",
      " 43%|████▎     | 109724/253378 [00:26<00:34, 4138.94it/s]\u001b[A\n",
      " 43%|████▎     | 110143/253378 [00:27<00:34, 4151.62it/s]\u001b[A\n",
      " 44%|████▎     | 110559/253378 [00:27<00:34, 4148.23it/s]\u001b[A\n",
      " 44%|████▍     | 110978/253378 [00:27<00:34, 4157.84it/s]\u001b[A\n",
      " 44%|████▍     | 111394/253378 [00:27<00:34, 4154.24it/s]\u001b[A\n",
      " 44%|████▍     | 111810/253378 [00:27<00:34, 4117.48it/s]\u001b[A\n",
      " 44%|████▍     | 112225/253378 [00:27<00:34, 4125.67it/s]\u001b[A\n",
      " 44%|████▍     | 112643/253378 [00:27<00:33, 4141.81it/s]\u001b[A\n",
      " 45%|████▍     | 113058/253378 [00:27<00:33, 4140.18it/s]\u001b[A\n",
      " 45%|████▍     | 113477/253378 [00:27<00:33, 4152.67it/s]\u001b[A\n",
      " 45%|████▍     | 113893/253378 [00:27<00:33, 4149.85it/s]\u001b[A\n",
      " 45%|████▌     | 114313/253378 [00:28<00:33, 4163.54it/s]\u001b[A\n",
      " 45%|████▌     | 114730/253378 [00:28<00:33, 4164.89it/s]\u001b[A\n",
      " 45%|████▌     | 115150/253378 [00:28<00:33, 4173.46it/s]\u001b[A\n",
      " 46%|████▌     | 115568/253378 [00:28<00:33, 4166.80it/s]\u001b[A\n",
      " 46%|████▌     | 115985/253378 [00:28<00:32, 4165.18it/s]\u001b[A\n",
      " 46%|████▌     | 116402/253378 [00:28<00:32, 4160.37it/s]\u001b[A\n",
      " 46%|████▌     | 116820/253378 [00:28<00:32, 4166.22it/s]\u001b[A\n",
      " 46%|████▋     | 117237/253378 [00:28<00:32, 4161.28it/s]\u001b[A\n",
      " 46%|████▋     | 117654/253378 [00:28<00:32, 4155.33it/s]\u001b[A\n",
      " 47%|████▋     | 118070/253378 [00:28<00:32, 4153.72it/s]\u001b[A\n",
      " 47%|████▋     | 118489/253378 [00:29<00:32, 4164.04it/s]\u001b[A\n",
      " 47%|████▋     | 118906/253378 [00:29<00:32, 4120.63it/s]\u001b[A\n",
      " 47%|████▋     | 119324/253378 [00:29<00:32, 4135.79it/s]\u001b[A\n",
      " 47%|████▋     | 119739/253378 [00:29<00:32, 4139.36it/s]\u001b[A\n",
      " 47%|████▋     | 120157/253378 [00:29<00:32, 4148.51it/s]\u001b[A\n",
      " 48%|████▊     | 120573/253378 [00:29<00:31, 4151.03it/s]\u001b[A\n",
      " 48%|████▊     | 120991/253378 [00:29<00:31, 4156.92it/s]\u001b[A\n",
      " 48%|████▊     | 121407/253378 [00:29<00:31, 4156.17it/s]\u001b[A\n",
      " 48%|████▊     | 121824/253378 [00:29<00:31, 4159.33it/s]\u001b[A\n",
      " 48%|████▊     | 122240/253378 [00:29<00:31, 4151.03it/s]\u001b[A\n",
      " 48%|████▊     | 122658/253378 [00:30<00:31, 4158.89it/s]\u001b[A\n",
      " 49%|████▊     | 123074/253378 [00:30<00:31, 4155.94it/s]\u001b[A\n",
      " 49%|████▊     | 123493/253378 [00:30<00:31, 4165.30it/s]\u001b[A\n",
      " 49%|████▉     | 123910/253378 [00:30<00:31, 4120.24it/s]\u001b[A\n",
      " 49%|████▉     | 124328/253378 [00:30<00:31, 4137.24it/s]\u001b[A\n",
      " 49%|████▉     | 124744/253378 [00:30<00:31, 4143.98it/s]\u001b[A\n",
      " 49%|████▉     | 125162/253378 [00:30<00:30, 4152.25it/s]\u001b[A\n",
      " 50%|████▉     | 125578/253378 [00:30<00:30, 4142.92it/s]\u001b[A\n",
      " 50%|████▉     | 125993/253378 [00:30<00:30, 4140.22it/s]\u001b[A\n",
      " 50%|████▉     | 126409/253378 [00:30<00:30, 4143.25it/s]\u001b[A\n",
      " 50%|█████     | 126828/253378 [00:31<00:30, 4155.52it/s]\u001b[A\n",
      " 50%|█████     | 127244/253378 [00:31<00:30, 4152.79it/s]\u001b[A\n",
      " 50%|█████     | 127664/253378 [00:31<00:30, 4165.46it/s]\u001b[A\n",
      " 51%|█████     | 128081/253378 [00:31<00:30, 4162.40it/s]\u001b[A\n",
      " 51%|█████     | 128499/253378 [00:31<00:29, 4166.91it/s]\u001b[A\n",
      " 51%|█████     | 128916/253378 [00:31<00:29, 4160.57it/s]\u001b[A\n",
      " 51%|█████     | 129333/253378 [00:31<00:29, 4156.53it/s]\u001b[A\n",
      " 51%|█████     | 129749/253378 [00:31<00:29, 4138.14it/s]\u001b[A\n",
      " 51%|█████▏    | 130163/253378 [00:31<00:29, 4137.47it/s]\u001b[A\n",
      " 52%|█████▏    | 130577/253378 [00:31<00:29, 4127.68it/s]\u001b[A\n",
      " 52%|█████▏    | 130993/253378 [00:32<00:29, 4136.96it/s]\u001b[A\n",
      " 52%|█████▏    | 131407/253378 [00:32<00:29, 4129.99it/s]\u001b[A\n",
      " 52%|█████▏    | 131823/253378 [00:32<00:29, 4137.94it/s]\u001b[A\n",
      " 52%|█████▏    | 132237/253378 [00:32<00:29, 4124.41it/s]\u001b[A\n",
      " 52%|█████▏    | 132650/253378 [00:32<00:29, 4048.89it/s]\u001b[A\n",
      " 53%|█████▎    | 133060/253378 [00:32<00:29, 4062.45it/s]\u001b[A\n",
      " 53%|█████▎    | 133475/253378 [00:32<00:29, 4086.22it/s]\u001b[A\n",
      " 53%|█████▎    | 133886/253378 [00:32<00:29, 4090.71it/s]\u001b[A\n",
      " 53%|█████▎    | 134296/253378 [00:32<00:29, 4081.88it/s]\u001b[A\n",
      " 53%|█████▎    | 134709/253378 [00:32<00:28, 4094.37it/s]\u001b[A\n",
      " 53%|█████▎    | 135119/253378 [00:33<00:28, 4092.53it/s]\u001b[A\n",
      " 53%|█████▎    | 135529/253378 [00:33<00:28, 4093.28it/s]\u001b[A\n",
      " 54%|█████▎    | 135939/253378 [00:33<00:28, 4093.18it/s]\u001b[A\n",
      " 54%|█████▍    | 136350/253378 [00:33<00:28, 4096.69it/s]\u001b[A\n",
      " 54%|█████▍    | 136762/253378 [00:33<00:28, 4102.39it/s]\u001b[A\n",
      " 54%|█████▍    | 137174/253378 [00:33<00:28, 4106.86it/s]\u001b[A\n",
      " 54%|█████▍    | 137585/253378 [00:33<00:28, 4104.46it/s]\u001b[A\n",
      " 54%|█████▍    | 137999/253378 [00:33<00:28, 4112.78it/s]\u001b[A\n",
      " 55%|█████▍    | 138411/253378 [00:33<00:27, 4108.78it/s]\u001b[A\n",
      " 55%|█████▍    | 138822/253378 [00:33<00:28, 4056.69it/s]\u001b[A\n",
      " 55%|█████▍    | 139233/253378 [00:34<00:28, 4070.20it/s]\u001b[A\n",
      " 55%|█████▌    | 139649/253378 [00:34<00:27, 4094.89it/s]\u001b[A\n",
      " 55%|█████▌    | 140061/253378 [00:34<00:27, 4099.98it/s]\u001b[A\n",
      " 55%|█████▌    | 140474/253378 [00:34<00:27, 4108.36it/s]\u001b[A\n",
      " 56%|█████▌    | 140885/253378 [00:34<00:27, 4104.91it/s]\u001b[A\n",
      " 56%|█████▌    | 141296/253378 [00:34<00:27, 4105.44it/s]\u001b[A\n",
      " 56%|█████▌    | 141707/253378 [00:34<00:27, 4100.09it/s]\u001b[A\n",
      " 56%|█████▌    | 142118/253378 [00:34<00:27, 4102.31it/s]\u001b[A\n",
      " 56%|█████▋    | 142529/253378 [00:34<00:27, 4097.25it/s]\u001b[A\n",
      " 56%|█████▋    | 142944/253378 [00:34<00:26, 4112.11it/s]\u001b[A\n",
      " 57%|█████▋    | 143356/253378 [00:35<00:26, 4112.23it/s]\u001b[A\n",
      " 57%|█████▋    | 143771/253378 [00:35<00:26, 4122.74it/s]\u001b[A\n",
      " 57%|█████▋    | 144184/253378 [00:35<00:26, 4112.92it/s]\u001b[A\n",
      " 57%|█████▋    | 144599/253378 [00:35<00:26, 4123.22it/s]\u001b[A\n",
      " 57%|█████▋    | 145012/253378 [00:35<00:26, 4055.21it/s]\u001b[A\n",
      " 57%|█████▋    | 145426/253378 [00:35<00:26, 4079.88it/s]\u001b[A\n",
      " 58%|█████▊    | 145837/253378 [00:35<00:26, 4086.08it/s]\u001b[A\n",
      " 58%|█████▊    | 146251/253378 [00:35<00:26, 4100.79it/s]\u001b[A\n",
      " 58%|█████▊    | 146663/253378 [00:35<00:26, 4103.71it/s]\u001b[A\n",
      " 58%|█████▊    | 147074/253378 [00:35<00:25, 4096.73it/s]\u001b[A\n",
      " 58%|█████▊    | 147484/253378 [00:36<00:25, 4081.54it/s]\u001b[A\n",
      " 58%|█████▊    | 147893/253378 [00:36<00:25, 4068.69it/s]\u001b[A\n",
      " 59%|█████▊    | 148300/253378 [00:36<00:25, 4062.42it/s]\u001b[A\n",
      " 59%|█████▊    | 148709/253378 [00:36<00:25, 4069.06it/s]\u001b[A\n",
      " 59%|█████▉    | 149116/253378 [00:36<00:25, 4063.19it/s]\u001b[A\n",
      " 59%|█████▉    | 149524/253378 [00:36<00:25, 4067.55it/s]\u001b[A\n",
      " 59%|█████▉    | 149931/253378 [00:36<00:25, 4059.98it/s]\u001b[A\n",
      " 59%|█████▉    | 150341/253378 [00:36<00:25, 4068.78it/s]\u001b[A\n",
      " 59%|█████▉    | 150748/253378 [00:36<00:25, 4064.61it/s]\u001b[A\n",
      " 60%|█████▉    | 151155/253378 [00:36<00:25, 3998.20it/s]\u001b[A\n",
      " 60%|█████▉    | 151560/253378 [00:37<00:25, 4011.29it/s]\u001b[A\n",
      " 60%|█████▉    | 151969/253378 [00:37<00:25, 4033.87it/s]\u001b[A\n",
      " 60%|██████    | 152374/253378 [00:37<00:25, 4036.65it/s]\u001b[A\n",
      " 60%|██████    | 152782/253378 [00:37<00:24, 4049.53it/s]\u001b[A\n",
      " 60%|██████    | 153188/253378 [00:37<00:24, 4048.46it/s]\u001b[A\n",
      " 61%|██████    | 153596/253378 [00:37<00:24, 4055.98it/s]\u001b[A\n",
      " 61%|██████    | 154002/253378 [00:37<00:24, 4050.92it/s]\u001b[A\n",
      " 61%|██████    | 154408/253378 [00:37<00:24, 4051.55it/s]\u001b[A\n",
      " 61%|██████    | 154814/253378 [00:37<00:24, 4053.81it/s]\u001b[A\n",
      " 61%|██████▏   | 155223/253378 [00:37<00:24, 4063.76it/s]\u001b[A\n",
      " 61%|██████▏   | 155630/253378 [00:38<00:24, 4057.32it/s]\u001b[A\n",
      " 62%|██████▏   | 156039/253378 [00:38<00:23, 4064.92it/s]\u001b[A\n",
      " 62%|██████▏   | 156446/253378 [00:38<00:23, 4060.92it/s]\u001b[A\n",
      " 62%|██████▏   | 156853/253378 [00:38<00:23, 4063.37it/s]\u001b[A\n",
      " 62%|██████▏   | 157260/253378 [00:38<00:24, 4004.28it/s]\u001b[A\n",
      " 62%|██████▏   | 157666/253378 [00:38<00:23, 4020.52it/s]\u001b[A\n",
      " 62%|██████▏   | 158072/253378 [00:38<00:23, 4030.58it/s]\u001b[A\n",
      " 63%|██████▎   | 158481/253378 [00:38<00:23, 4046.14it/s]\u001b[A\n",
      " 63%|██████▎   | 158886/253378 [00:38<00:23, 4042.88it/s]\u001b[A\n",
      " 63%|██████▎   | 159291/253378 [00:38<00:23, 4037.45it/s]\u001b[A\n",
      " 63%|██████▎   | 159698/253378 [00:39<00:23, 4044.98it/s]\u001b[A\n",
      " 63%|██████▎   | 160104/253378 [00:39<00:23, 4048.13it/s]\u001b[A\n",
      " 63%|██████▎   | 160509/253378 [00:39<00:22, 4041.74it/s]\u001b[A\n",
      " 64%|██████▎   | 160916/253378 [00:39<00:22, 4049.52it/s]\u001b[A\n",
      " 64%|██████▎   | 161321/253378 [00:39<00:22, 4046.51it/s]\u001b[A\n",
      " 64%|██████▍   | 161730/253378 [00:39<00:22, 4057.87it/s]\u001b[A\n",
      " 64%|██████▍   | 162136/253378 [00:39<00:22, 4056.10it/s]\u001b[A\n",
      " 64%|██████▍   | 162543/253378 [00:39<00:22, 4060.23it/s]\u001b[A\n",
      " 64%|██████▍   | 162950/253378 [00:39<00:22, 4056.21it/s]\u001b[A\n",
      " 64%|██████▍   | 163356/253378 [00:39<00:22, 4004.70it/s]\u001b[A\n",
      " 65%|██████▍   | 163761/253378 [00:40<00:22, 4016.40it/s]\u001b[A\n",
      " 65%|██████▍   | 164168/253378 [00:40<00:22, 4030.94it/s]\u001b[A\n",
      " 65%|██████▍   | 164574/253378 [00:40<00:21, 4036.92it/s]\u001b[A\n",
      " 65%|██████▌   | 164982/253378 [00:40<00:21, 4049.20it/s]\u001b[A\n",
      " 65%|██████▌   | 165388/253378 [00:40<00:21, 4049.71it/s]\u001b[A\n",
      " 65%|██████▌   | 165797/253378 [00:40<00:21, 4058.48it/s]\u001b[A\n",
      " 66%|██████▌   | 166203/253378 [00:40<00:21, 4052.04it/s]\u001b[A\n",
      " 66%|██████▌   | 166609/253378 [00:40<00:21, 4028.09it/s]\u001b[A\n",
      " 66%|██████▌   | 167020/253378 [00:40<00:21, 4050.23it/s]\u001b[A\n",
      " 66%|██████▌   | 167431/253378 [00:40<00:21, 4067.76it/s]\u001b[A\n",
      " 66%|██████▌   | 167838/253378 [00:41<00:21, 4065.66it/s]\u001b[A\n",
      " 66%|██████▋   | 168252/253378 [00:41<00:20, 4087.55it/s]\u001b[A\n",
      " 67%|██████▋   | 168663/253378 [00:41<00:20, 4091.62it/s]\u001b[A\n",
      " 67%|██████▋   | 169074/253378 [00:41<00:20, 4094.32it/s]\u001b[A\n",
      " 67%|██████▋   | 169484/253378 [00:41<00:20, 4038.96it/s]\u001b[A\n",
      " 67%|██████▋   | 169897/253378 [00:41<00:20, 4063.39it/s]\u001b[A\n",
      " 67%|██████▋   | 170304/253378 [00:41<00:20, 4064.77it/s]\u001b[A\n",
      " 67%|██████▋   | 170716/253378 [00:41<00:20, 4078.68it/s]\u001b[A\n",
      " 68%|██████▊   | 171127/253378 [00:41<00:20, 4085.50it/s]\u001b[A\n",
      " 68%|██████▊   | 171542/253378 [00:41<00:19, 4101.79it/s]\u001b[A\n",
      " 68%|██████▊   | 171953/253378 [00:42<00:19, 4103.90it/s]\u001b[A\n",
      " 68%|██████▊   | 172366/253378 [00:42<00:19, 4110.50it/s]\u001b[A\n",
      " 68%|██████▊   | 172778/253378 [00:42<00:19, 4106.66it/s]\u001b[A\n",
      " 68%|██████▊   | 173190/253378 [00:42<00:19, 4108.88it/s]\u001b[A\n",
      " 69%|██████▊   | 173601/253378 [00:42<00:19, 4107.08it/s]\u001b[A\n",
      " 69%|██████▊   | 174015/253378 [00:42<00:19, 4116.37it/s]\u001b[A\n",
      " 69%|██████▉   | 174427/253378 [00:42<00:19, 4116.22it/s]\u001b[A\n",
      " 69%|██████▉   | 174839/253378 [00:42<00:19, 4044.51it/s]\u001b[A\n",
      " 69%|██████▉   | 175250/253378 [00:42<00:19, 4063.65it/s]\u001b[A\n",
      " 69%|██████▉   | 175657/253378 [00:43<00:19, 4024.89it/s]\u001b[A\n",
      " 69%|██████▉   | 176065/253378 [00:43<00:19, 4040.33it/s]\u001b[A\n",
      " 70%|██████▉   | 176480/253378 [00:43<00:18, 4070.66it/s]\u001b[A\n",
      " 70%|██████▉   | 176890/253378 [00:43<00:18, 4077.86it/s]\u001b[A\n",
      " 70%|██████▉   | 177300/253378 [00:43<00:18, 4084.31it/s]\u001b[A\n",
      " 70%|███████   | 177716/253378 [00:43<00:18, 4106.44it/s]\u001b[A\n",
      " 70%|███████   | 178127/253378 [00:43<00:18, 4103.05it/s]\u001b[A\n",
      " 70%|███████   | 178540/253378 [00:43<00:18, 4108.86it/s]\u001b[A\n",
      " 71%|███████   | 178951/253378 [00:43<00:18, 4098.04it/s]\u001b[A\n",
      " 71%|███████   | 179362/253378 [00:43<00:18, 4101.04it/s]\u001b[A\n",
      " 71%|███████   | 179773/253378 [00:44<00:17, 4100.95it/s]\u001b[A\n",
      " 71%|███████   | 180187/253378 [00:44<00:17, 4110.43it/s]\u001b[A\n",
      " 71%|███████▏  | 180599/253378 [00:44<00:17, 4107.80it/s]\u001b[A\n",
      " 71%|███████▏  | 181013/253378 [00:44<00:17, 4116.84it/s]\u001b[A\n",
      " 72%|███████▏  | 181425/253378 [00:44<00:17, 4114.66it/s]\u001b[A\n",
      " 72%|███████▏  | 181837/253378 [00:44<00:17, 4058.77it/s]\u001b[A\n",
      " 72%|███████▏  | 182248/253378 [00:44<00:17, 4071.86it/s]\u001b[A\n",
      " 72%|███████▏  | 182665/253378 [00:44<00:17, 4098.08it/s]\u001b[A\n",
      " 72%|███████▏  | 183075/253378 [00:44<00:17, 4088.77it/s]\u001b[A\n",
      " 72%|███████▏  | 183491/253378 [00:44<00:17, 4107.49it/s]\u001b[A\n",
      " 73%|███████▎  | 183904/253378 [00:45<00:16, 4111.29it/s]\u001b[A\n",
      " 73%|███████▎  | 184317/253378 [00:45<00:16, 4114.01it/s]\u001b[A\n",
      " 73%|███████▎  | 184729/253378 [00:45<00:16, 4113.77it/s]\u001b[A\n",
      " 73%|███████▎  | 185144/253378 [00:45<00:16, 4121.95it/s]\u001b[A\n",
      " 73%|███████▎  | 185557/253378 [00:45<00:16, 4119.13it/s]\u001b[A\n",
      " 73%|███████▎  | 185970/253378 [00:45<00:16, 4121.24it/s]\u001b[A\n",
      " 74%|███████▎  | 186383/253378 [00:45<00:16, 4095.83it/s]\u001b[A\n",
      " 74%|███████▎  | 186798/253378 [00:45<00:16, 4110.31it/s]\u001b[A\n",
      " 74%|███████▍  | 187210/253378 [00:45<00:16, 4112.89it/s]\u001b[A\n",
      " 74%|███████▍  | 187626/253378 [00:45<00:15, 4124.75it/s]\u001b[A\n",
      " 74%|███████▍  | 188039/253378 [00:46<00:16, 4062.68it/s]\u001b[A\n",
      " 74%|███████▍  | 188454/253378 [00:46<00:15, 4087.13it/s]\u001b[A\n",
      " 75%|███████▍  | 188866/253378 [00:46<00:15, 4096.27it/s]\u001b[A\n",
      " 75%|███████▍  | 189282/253378 [00:46<00:15, 4112.37it/s]\u001b[A\n",
      " 75%|███████▍  | 189694/253378 [00:46<00:15, 4110.44it/s]\u001b[A\n",
      " 75%|███████▌  | 190106/253378 [00:46<00:15, 4084.10it/s]\u001b[A\n",
      " 75%|███████▌  | 190517/253378 [00:46<00:15, 4091.76it/s]\u001b[A\n",
      " 75%|███████▌  | 190933/253378 [00:46<00:15, 4109.50it/s]\u001b[A\n",
      " 76%|███████▌  | 191345/253378 [00:46<00:15, 4096.89it/s]\u001b[A\n",
      " 76%|███████▌  | 191760/253378 [00:46<00:14, 4110.32it/s]\u001b[A\n",
      " 76%|███████▌  | 192172/253378 [00:47<00:14, 4112.40it/s]\u001b[A\n",
      " 76%|███████▌  | 192584/253378 [00:47<00:14, 4113.07it/s]\u001b[A\n",
      " 76%|███████▌  | 192996/253378 [00:47<00:14, 4109.80it/s]\u001b[A\n",
      " 76%|███████▋  | 193411/253378 [00:47<00:14, 4121.21it/s]\u001b[A\n",
      " 76%|███████▋  | 193824/253378 [00:47<00:14, 4120.41it/s]\u001b[A\n",
      " 77%|███████▋  | 194239/253378 [00:47<00:14, 4127.02it/s]\u001b[A\n",
      " 77%|███████▋  | 194652/253378 [00:47<00:14, 4114.24it/s]\u001b[A\n",
      " 77%|███████▋  | 195068/253378 [00:47<00:14, 4125.25it/s]\u001b[A\n",
      " 77%|███████▋  | 195481/253378 [00:47<00:14, 4125.26it/s]\u001b[A\n",
      " 77%|███████▋  | 195895/253378 [00:47<00:13, 4128.97it/s]\u001b[A\n",
      " 77%|███████▋  | 196308/253378 [00:48<00:13, 4127.21it/s]\u001b[A\n",
      " 78%|███████▊  | 196723/253378 [00:48<00:13, 4133.18it/s]\u001b[A\n",
      " 78%|███████▊  | 197137/253378 [00:48<00:13, 4126.50it/s]\u001b[A\n",
      " 78%|███████▊  | 197553/253378 [00:48<00:13, 4133.87it/s]\u001b[A\n",
      " 78%|███████▊  | 197967/253378 [00:48<00:13, 4129.03it/s]\u001b[A\n",
      " 78%|███████▊  | 198382/253378 [00:48<00:13, 4134.32it/s]\u001b[A\n",
      " 78%|███████▊  | 198796/253378 [00:48<00:13, 4129.19it/s]\u001b[A\n",
      " 79%|███████▊  | 199209/253378 [00:48<00:13, 4121.94it/s]\u001b[A\n",
      " 79%|███████▉  | 199622/253378 [00:48<00:13, 4119.47it/s]\u001b[A\n",
      " 79%|███████▉  | 200034/253378 [00:48<00:12, 4115.07it/s]\u001b[A\n",
      " 79%|███████▉  | 200446/253378 [00:49<00:12, 4111.66it/s]\u001b[A\n",
      " 79%|███████▉  | 200858/253378 [00:49<00:12, 4113.20it/s]\u001b[A\n",
      " 79%|███████▉  | 201270/253378 [00:49<00:12, 4101.81it/s]\u001b[A\n",
      " 80%|███████▉  | 201685/253378 [00:49<00:12, 4114.14it/s]\u001b[A\n",
      " 80%|███████▉  | 202097/253378 [00:49<00:12, 4115.13it/s]\u001b[A\n",
      " 80%|███████▉  | 202511/253378 [00:49<00:12, 4121.18it/s]\u001b[A\n",
      " 80%|████████  | 202924/253378 [00:49<00:12, 4122.47it/s]\u001b[A\n",
      " 80%|████████  | 203340/253378 [00:49<00:12, 4132.41it/s]\u001b[A\n",
      " 80%|████████  | 203754/253378 [00:49<00:12, 4104.58it/s]\u001b[A\n",
      " 81%|████████  | 204170/253378 [00:49<00:11, 4118.27it/s]\u001b[A\n",
      " 81%|████████  | 204582/253378 [00:50<00:11, 4117.39it/s]\u001b[A\n",
      " 81%|████████  | 204997/253378 [00:50<00:11, 4124.86it/s]\u001b[A\n",
      " 81%|████████  | 205410/253378 [00:50<00:11, 4122.77it/s]\u001b[A\n",
      " 81%|████████  | 205823/253378 [00:50<00:11, 4122.68it/s]\u001b[A\n",
      " 81%|████████▏ | 206236/253378 [00:50<00:11, 4108.54it/s]\u001b[A\n",
      " 82%|████████▏ | 206650/253378 [00:50<00:11, 4116.40it/s]\u001b[A\n",
      " 82%|████████▏ | 207062/253378 [00:50<00:11, 4114.18it/s]\u001b[A\n",
      " 82%|████████▏ | 207477/253378 [00:50<00:11, 4122.87it/s]\u001b[A\n",
      " 82%|████████▏ | 207890/253378 [00:50<00:11, 4119.11it/s]\u001b[A\n",
      " 82%|████████▏ | 208305/253378 [00:50<00:10, 4126.33it/s]\u001b[A\n",
      " 82%|████████▏ | 208718/253378 [00:51<00:10, 4116.78it/s]\u001b[A\n",
      " 83%|████████▎ | 209134/253378 [00:51<00:10, 4129.02it/s]\u001b[A\n",
      " 83%|████████▎ | 209547/253378 [00:51<00:10, 4114.61it/s]\u001b[A\n",
      " 83%|████████▎ | 209961/253378 [00:51<00:10, 4118.91it/s]\u001b[A\n",
      " 83%|████████▎ | 210373/253378 [00:51<00:10, 4118.91it/s]\u001b[A\n",
      " 83%|████████▎ | 210789/253378 [00:51<00:10, 4129.92it/s]\u001b[A\n",
      " 83%|████████▎ | 211203/253378 [00:51<00:10, 4125.36it/s]\u001b[A\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 84%|████████▎ | 211617/253378 [00:51<00:10, 4128.81it/s]\u001b[A\n",
      " 84%|████████▎ | 212030/253378 [00:51<00:10, 4119.11it/s]\u001b[A\n",
      " 84%|████████▍ | 212445/253378 [00:51<00:09, 4127.48it/s]\u001b[A\n",
      " 84%|████████▍ | 212858/253378 [00:52<00:09, 4125.73it/s]\u001b[A\n",
      " 84%|████████▍ | 213271/253378 [00:52<00:09, 4125.37it/s]\u001b[A\n",
      " 84%|████████▍ | 213684/253378 [00:52<00:09, 4121.36it/s]\u001b[A\n",
      " 84%|████████▍ | 214097/253378 [00:52<00:09, 4118.57it/s]\u001b[A\n",
      " 85%|████████▍ | 214510/253378 [00:52<00:09, 4119.26it/s]\u001b[A\n",
      " 85%|████████▍ | 214922/253378 [00:52<00:09, 4081.33it/s]\u001b[A\n",
      " 85%|████████▍ | 215333/253378 [00:52<00:09, 4088.83it/s]\u001b[A\n",
      " 85%|████████▌ | 215749/253378 [00:52<00:09, 4108.52it/s]\u001b[A\n",
      " 85%|████████▌ | 216161/253378 [00:52<00:09, 4111.28it/s]\u001b[A\n",
      " 85%|████████▌ | 216576/253378 [00:52<00:08, 4122.09it/s]\u001b[A\n",
      " 86%|████████▌ | 216989/253378 [00:53<00:08, 4118.89it/s]\u001b[A\n",
      " 86%|████████▌ | 217401/253378 [00:53<00:08, 4116.31it/s]\u001b[A\n",
      " 86%|████████▌ | 217813/253378 [00:53<00:08, 4114.09it/s]\u001b[A\n",
      " 86%|████████▌ | 218225/253378 [00:53<00:08, 4115.35it/s]\u001b[A\n",
      " 86%|████████▋ | 218639/253378 [00:53<00:08, 4122.16it/s]\u001b[A\n",
      " 86%|████████▋ | 219052/253378 [00:53<00:08, 4116.91it/s]\u001b[A\n",
      " 87%|████████▋ | 219464/253378 [00:53<00:08, 4115.99it/s]\u001b[A\n",
      " 87%|████████▋ | 219876/253378 [00:53<00:08, 4115.56it/s]\u001b[A\n",
      " 87%|████████▋ | 220292/253378 [00:53<00:08, 4126.04it/s]\u001b[A\n",
      " 87%|████████▋ | 220705/253378 [00:53<00:07, 4124.45it/s]\u001b[A\n",
      " 87%|████████▋ | 221121/253378 [00:54<00:07, 4132.62it/s]\u001b[A\n",
      " 87%|████████▋ | 221535/253378 [00:54<00:07, 4102.76it/s]\u001b[A\n",
      " 88%|████████▊ | 221950/253378 [00:54<00:07, 4114.91it/s]\u001b[A\n",
      " 88%|████████▊ | 222362/253378 [00:54<00:07, 4109.73it/s]\u001b[A\n",
      " 88%|████████▊ | 222777/253378 [00:54<00:07, 4120.65it/s]\u001b[A\n",
      " 88%|████████▊ | 223190/253378 [00:54<00:07, 4116.64it/s]\u001b[A\n",
      " 88%|████████▊ | 223604/253378 [00:54<00:07, 4123.00it/s]\u001b[A\n",
      " 88%|████████▊ | 224017/253378 [00:54<00:07, 4117.85it/s]\u001b[A\n",
      " 89%|████████▊ | 224431/253378 [00:54<00:07, 4122.57it/s]\u001b[A\n",
      " 89%|████████▊ | 224844/253378 [00:54<00:06, 4115.46it/s]\u001b[A\n",
      " 89%|████████▉ | 225259/253378 [00:55<00:06, 4125.14it/s]\u001b[A\n",
      " 89%|████████▉ | 225672/253378 [00:55<00:06, 4119.47it/s]\u001b[A\n",
      " 89%|████████▉ | 226084/253378 [00:55<00:06, 4118.13it/s]\u001b[A\n",
      " 89%|████████▉ | 226496/253378 [00:55<00:06, 4115.42it/s]\u001b[A\n",
      " 90%|████████▉ | 226908/253378 [00:55<00:06, 4114.72it/s]\u001b[A\n",
      " 90%|████████▉ | 227320/253378 [00:55<00:06, 4112.68it/s]\u001b[A\n",
      " 90%|████████▉ | 227736/253378 [00:55<00:06, 4125.11it/s]\u001b[A\n",
      " 90%|█████████ | 228149/253378 [00:55<00:06, 4117.25it/s]\u001b[A\n",
      " 90%|█████████ | 228564/253378 [00:55<00:06, 4126.80it/s]\u001b[A\n",
      " 90%|█████████ | 228977/253378 [00:55<00:05, 4124.03it/s]\u001b[A\n",
      " 91%|█████████ | 229391/253378 [00:56<00:05, 4128.29it/s]\u001b[A\n",
      " 91%|█████████ | 229804/253378 [00:56<00:05, 4118.91it/s]\u001b[A\n",
      " 91%|█████████ | 230218/253378 [00:56<00:05, 4122.21it/s]\u001b[A\n",
      " 91%|█████████ | 230631/253378 [00:56<00:05, 4118.76it/s]\u001b[A\n",
      " 91%|█████████ | 231045/253378 [00:56<00:05, 4124.37it/s]\u001b[A\n",
      " 91%|█████████▏| 231458/253378 [00:56<00:05, 4118.86it/s]\u001b[A\n",
      " 92%|█████████▏| 231874/253378 [00:56<00:05, 4129.51it/s]\u001b[A\n",
      " 92%|█████████▏| 232287/253378 [00:56<00:05, 4123.95it/s]\u001b[A\n",
      " 92%|█████████▏| 232702/253378 [00:56<00:05, 4130.44it/s]\u001b[A\n",
      " 92%|█████████▏| 233116/253378 [00:56<00:04, 4119.28it/s]\u001b[A\n",
      " 92%|█████████▏| 233532/253378 [00:57<00:04, 4129.19it/s]\u001b[A\n",
      " 92%|█████████▏| 233945/253378 [00:57<00:04, 4123.09it/s]\u001b[A\n",
      " 92%|█████████▏| 234358/253378 [00:57<00:04, 4114.37it/s]\u001b[A\n",
      " 93%|█████████▎| 234770/253378 [00:57<00:04, 4112.92it/s]\u001b[A\n",
      " 93%|█████████▎| 235185/253378 [00:57<00:04, 4122.62it/s]\u001b[A\n",
      " 93%|█████████▎| 235598/253378 [00:57<00:04, 4118.28it/s]\u001b[A\n",
      " 93%|█████████▎| 236013/253378 [00:57<00:04, 4126.22it/s]\u001b[A\n",
      " 93%|█████████▎| 236426/253378 [00:57<00:04, 4109.53it/s]\u001b[A\n",
      " 93%|█████████▎| 236841/253378 [00:57<00:04, 4121.02it/s]\u001b[A\n",
      " 94%|█████████▎| 237254/253378 [00:57<00:03, 4102.77it/s]\u001b[A\n",
      " 94%|█████████▍| 237668/253378 [00:58<00:03, 4112.80it/s]\u001b[A\n",
      " 94%|█████████▍| 238081/253378 [00:58<00:03, 4115.87it/s]\u001b[A\n",
      " 94%|█████████▍| 238496/253378 [00:58<00:03, 4125.72it/s]\u001b[A\n",
      " 94%|█████████▍| 238909/253378 [00:58<00:03, 4119.93it/s]\u001b[A\n",
      " 94%|█████████▍| 239324/253378 [00:58<00:03, 4126.75it/s]\u001b[A\n",
      " 95%|█████████▍| 239737/253378 [00:58<00:03, 4120.86it/s]\u001b[A\n",
      " 95%|█████████▍| 240150/253378 [00:58<00:03, 4117.83it/s]\u001b[A\n",
      " 95%|█████████▍| 240562/253378 [00:58<00:03, 4114.20it/s]\u001b[A\n",
      " 95%|█████████▌| 240975/253378 [00:58<00:03, 4116.91it/s]\u001b[A\n",
      " 95%|█████████▌| 241387/253378 [00:58<00:02, 4107.14it/s]\u001b[A\n",
      " 95%|█████████▌| 241802/253378 [00:59<00:02, 4118.07it/s]\u001b[A\n",
      " 96%|█████████▌| 242214/253378 [00:59<00:02, 4106.57it/s]\u001b[A\n",
      " 96%|█████████▌| 242630/253378 [00:59<00:02, 4119.83it/s]\u001b[A\n",
      " 96%|█████████▌| 243043/253378 [00:59<00:02, 4100.65it/s]\u001b[A\n",
      " 96%|█████████▌| 243458/253378 [00:59<00:02, 4113.51it/s]\u001b[A\n",
      " 96%|█████████▌| 243870/253378 [00:59<00:02, 4104.01it/s]\u001b[A\n",
      " 96%|█████████▋| 244284/253378 [00:59<00:02, 4113.81it/s]\u001b[A\n",
      " 97%|█████████▋| 244696/253378 [00:59<00:02, 4113.92it/s]\u001b[A\n",
      " 97%|█████████▋| 245111/253378 [00:59<00:02, 4124.25it/s]\u001b[A\n",
      " 97%|█████████▋| 245524/253378 [00:59<00:01, 4115.67it/s]\u001b[A\n",
      " 97%|█████████▋| 245938/253378 [01:00<00:01, 4120.61it/s]\u001b[A\n",
      " 97%|█████████▋| 246351/253378 [01:00<00:01, 4112.58it/s]\u001b[A\n",
      " 97%|█████████▋| 246765/253378 [01:00<00:01, 4118.20it/s]\u001b[A\n",
      " 98%|█████████▊| 247177/253378 [01:00<00:01, 4106.00it/s]\u001b[A\n",
      " 98%|█████████▊| 247591/253378 [01:00<00:01, 4115.10it/s]\u001b[A\n",
      " 98%|█████████▊| 248003/253378 [01:00<00:01, 4112.72it/s]\u001b[A\n",
      " 98%|█████████▊| 248419/253378 [01:00<00:01, 4126.31it/s]\u001b[A\n",
      " 98%|█████████▊| 248832/253378 [01:00<00:01, 4122.27it/s]\u001b[A\n",
      " 98%|█████████▊| 249246/253378 [01:00<00:01, 4127.18it/s]\u001b[A\n",
      " 99%|█████████▊| 249659/253378 [01:00<00:00, 4083.68it/s]\u001b[A\n",
      " 99%|█████████▊| 250073/253378 [01:01<00:00, 4099.24it/s]\u001b[A\n",
      " 99%|█████████▉| 250484/253378 [01:01<00:00, 4091.08it/s]\u001b[A\n",
      " 99%|█████████▉| 250896/253378 [01:01<00:00, 4098.36it/s]\u001b[A\n",
      " 99%|█████████▉| 251306/253378 [01:01<00:00, 4087.51it/s]\u001b[A\n",
      " 99%|█████████▉| 251718/253378 [01:01<00:00, 4094.04it/s]\u001b[A\n",
      "100%|█████████▉| 252131/253378 [01:01<00:00, 4102.26it/s]\u001b[A\n",
      "100%|█████████▉| 252547/253378 [01:01<00:00, 4117.82it/s]\u001b[A\n",
      "100%|█████████▉| 252960/253378 [01:01<00:00, 4120.57it/s]\u001b[A\n",
      "100%|██████████| 253378/253378 [01:01<00:00, 4094.19it/s]\u001b[A\n"
     ]
    }
   ],
   "source": [
    "%run ../twitter15/userdata_parser.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "  0%|          | 0/253378 [00:00<?, ?it/s]\u001b[A\n",
      " 10%|▉         | 24205/253378 [00:00<00:00, 242048.50it/s]\u001b[A\n",
      " 22%|██▏       | 56778/253378 [00:00<00:00, 262259.88it/s]\u001b[A\n",
      " 37%|███▋      | 92639/253378 [00:00<00:00, 285250.82it/s]\u001b[A\n",
      " 48%|████▊     | 122106/253378 [00:00<00:00, 288012.41it/s]\u001b[A\n",
      " 62%|██████▏   | 157030/253378 [00:00<00:00, 304000.68it/s]\u001b[A\n",
      " 76%|███████▌  | 192015/253378 [00:00<00:00, 316441.67it/s]\u001b[A\n",
      "100%|██████████| 253378/253378 [00:00<00:00, 325551.51it/s]\u001b[A\n"
     ]
    }
   ],
   "source": [
    "for key in tqdm(userVects):\n",
    "    userVects[key] = userVects[key].float()\n",
    "\n",
    "userVects = defaultdict(lambda:torch.tensor([1.1100e+02, 1.5000e+01, 0.0000e+00, 7.9700e+02, 4.7300e+02, 0.0000e+00,\n",
    "        8.3326e+04, 1.0000e+00]),userVects)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "userVects[0].requires_grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0107 10:05:24.364788 139830510675712 file_utils.py:39] PyTorch version 1.3.0 available.\n",
      "I0107 10:05:24.399865 139830510675712 modeling_xlnet.py:194] Better speed can be achieved with apex installed from https://www.github.com/nvidia/apex .\n"
     ]
    }
   ],
   "source": [
    "%run ./textEncoders.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%run ./temporal_tree_model.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "if torch.cuda.is_available():\n",
    "    device = 'cuda:2'\n",
    "    device = 'cpu'\n",
    "else:\n",
    "    device = 'cpu'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'false': 0, 'true': 1, 'unverified': 2, 'non-rumor': 3}"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labelMap = {}\n",
    "labelCount = 0\n",
    "for label in list(twitter15_labels.values()):\n",
    "    if label not in labelMap:\n",
    "        labelMap[label] = labelCount\n",
    "        labelCount += 1\n",
    "labelMap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'twitter15_trees' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-26-eb8c0b0d984e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mX_text\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mtid\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtwitter15_trees\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mtid\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtwitter15_trees\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mtid\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtwitter15_labels\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m             \u001b[0mX\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtuple\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtwitter15_trees\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtid\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mtwitter15_text\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtid\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'twitter15_trees' is not defined"
     ]
    }
   ],
   "source": [
    "epochs = 10\n",
    "X = []\n",
    "y = []\n",
    "X_text = []\n",
    "\n",
    "for tid in twitter15_trees:\n",
    "        if tid in twitter15_trees and tid in twitter15_labels:\n",
    "            X.append(tuple((twitter15_trees[tid],twitter15_text[tid])))\n",
    "            y.append(labelMap[twitter15_labels[tid]])\n",
    "            X_text.append(twitter15_text[tid])\n",
    "            \n",
    "x_train,x_test,y_train,y_test = train_test_split(X,y,random_state=2018)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class jointModel(nn.Module):\n",
    "    def __init__(self, treeEncoderType, textEncoderType, pretrain, textparams, treeparams, X, y, device):\n",
    "        super(jointModel, self).__init__()\n",
    "        if textEncoderType == 'rnn':\n",
    "            self.textEncoderModel = TextEncoder(textEncoderType,textparams,X,y,device)\n",
    "            textparams['hidden_dim'] = textparams['hidden_dim']*self.textEncoderModel.textEncoder.numDirs\n",
    "            \n",
    "        if textEncoderType == 'bert':\n",
    "            self.textEncoderModel = BertTextEncoder(textEncoderType,{},X,y,device)\n",
    "            textparams['hidden_dim'] = 768\n",
    "        \n",
    "        if textEncoderType == 'attn':\n",
    "            self.textEncoderModel = AttentionTextEncoder(textEncoderType,textparams,X,y,device)\n",
    "            textparams['hidden_dim'] = textparams['hidden_dim']*self.textEncoderModel.textEncoder.numDirs*self.textEncoderModel.seq_dim\n",
    "        \n",
    "        if pretrain:\n",
    "            self.textEncoderModel.trainModel()\n",
    "            self.textEncoderModel.textEncoder.load_state_dict(self.textEncoderModel.optimalParams)\n",
    "            \n",
    "        if treeEncoderType == 'standard':\n",
    "            self.treeEncoderModel = treeEncoder(treeparams['cuda'],treeparams['in_dim'],treeparams['mem_dim'],treeparams['userVects'],treeparams['labels'],treeparams['labelMap'],treeparams['criterion'],device)\n",
    "        if treeEncoderType == 'decay':\n",
    "            self.treeEncoderModel = decayTreeEncoder(treeparams['cuda'],treeparams['in_dim'],treeparams['mem_dim'],treeparams['userVects'],treeparams['labels'],treeparams['labelMap'],treeparams['criterion'],device)\n",
    "        \n",
    "        mem_dim = treeparams['mem_dim'] + textparams['hidden_dim']\n",
    "        \n",
    "        self.fc = nn.Linear(mem_dim,4)    \n",
    "            \n",
    "    def forward(self,tree,text):\n",
    "        treeVec = self.treeEncoderModel(tree)\n",
    "        treeVec = treeVec[0][1].reshape(-1)\n",
    "        \n",
    "        self.textEncoderModel.textEncoder = self.textEncoderModel.textEncoder.to('cpu')\n",
    "        textVec = self.textEncoderModel(text)\n",
    "        textVec = textVec.reshape(-1)\n",
    "#         print(treeVec.shape)\n",
    "#         print(textVec.shape)\n",
    "        combVec =  torch.cat((treeVec,textVec))\n",
    "#         combVec = textVec\n",
    "        out = self.fc(combVec)\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def trainModel(model,modelname):\n",
    "#     optimizer = torch.optim.Adagrad(model.treeEncoderModel.parameters(),0.01)\n",
    "    \n",
    "#     bertoptimizer = AdamW(model.textEncoderModel.parameters(),\n",
    "#                   lr = 2e-5, # args.learning_rate - default is 5e-5, our notebook had 2e-5\n",
    "#                   eps = 1e-8 # args.adam_epsilon  - default is 1e-8.\n",
    "#                 )\n",
    "\n",
    "    optimizer = torch.optim.Adagrad(model.parameters(),0.01)\n",
    "    \n",
    "    criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "    maxAcc = 0\n",
    "    count = 0\n",
    "    \n",
    "    for i in range(10):\n",
    "        for treeSet, text in tqdm(x_train):\n",
    "            tree = treeSet[-1]\n",
    "            optimizer.zero_grad()\n",
    "            \n",
    "            pred = model(tree.root,text)\n",
    "            \n",
    "            label = Variable(torch.tensor(labelMap[treeSet[0].root.label]).reshape(-1).to(device))\n",
    "            loss = criterion(pred.reshape(1,4),label)\n",
    "#             print(loss)\n",
    "    \n",
    "            if count % 20 == 0:\n",
    "#                 print('opt')\n",
    "                loss.backward()\n",
    "                optimizer.step()\n",
    "        \n",
    "        preds = []\n",
    "        labels = []\n",
    "\n",
    "        allLabels = []\n",
    "        allPreds = []\n",
    "        \n",
    "        for valSet, text in tqdm(x_test):\n",
    "            finalTree = valSet[-1]\n",
    "\n",
    "            predicted = model(finalTree.root,text)\n",
    "            preds.append(predicted)\n",
    "    #         print(predicted)\n",
    "            predicted =  torch.softmax(predicted,0)\n",
    "            predicted = torch.max(predicted, 0)[1].cpu().numpy().tolist()\n",
    "\n",
    "            labels.append(labelMap[finalTree.root.label])\n",
    "\n",
    "            allLabels.append(labelMap[finalTree.root.label])\n",
    "            allPreds.append(predicted)\n",
    "\n",
    "        predTensor = torch.stack(preds)\n",
    "        labelTensor = torch.tensor(labels).to(device)\n",
    "\n",
    "        print(allLabels,allPreds)\n",
    "\n",
    "        loss = criterion(predTensor.reshape(-1,4), labelTensor.reshape(-1))\n",
    "\n",
    "        cr = classification_report(allLabels,allPreds,output_dict=True)\n",
    "        cr['loss'] = loss.item()\n",
    "        cr['Acc'] = accuracy_score(allLabels,allPreds,)\n",
    "\n",
    "        if cr['Acc'] > maxAcc:\n",
    "            maxAcc = cr['Acc']\n",
    "#             torch.save({'state_dict': model.state_dict()}, './jointlyTrainedResults/'+modelname+'.pth')\n",
    "\n",
    "        print('loss: ',cr['loss'])\n",
    "        print(cr['Acc'])\n",
    "\n",
    "        with open('./jointlyTrainedResults/'+modelname+'json', 'a') as fp:\n",
    "            json.dump(cr, fp)\n",
    "            fp.write('\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "treeparams = {\n",
    "    'cuda': torch.cuda.is_available(),\n",
    "    'in_dim':8,\n",
    "    'mem_dim':100,\n",
    "    'userVects':userVects,\n",
    "    'labels':twitter15_labels,\n",
    "    'labelMap':labelMap,\n",
    "    'criterion':nn.CrossEntropyLoss()\n",
    "}\n",
    "\n",
    "textparams = {\n",
    "    'embedding_dim':256,\n",
    "    'hidden_dim': 50,\n",
    "    'output_dim':4,\n",
    "    'num_layers':1,\n",
    "    'bidir':True,\n",
    "    'rnnType':'gru'\n",
    "}\n",
    "\n",
    "treeTypes = ['standard','decay']\n",
    "textTypes = ['bert']\n",
    "pretrainTypes = [True,False]\n",
    "\n",
    "treeTypes = ['standard','decay']\n",
    "textTypes = ['bert']\n",
    "pretrainTypes = [True]\n",
    "\n",
    "for textType in textTypes:\n",
    "    for treeType in treeTypes:\n",
    "        for pretrainType in pretrainTypes:\n",
    "            model = jointModel(treeType,textType,pretrainType,textparams,treeparams,X_text,y,device)\n",
    "            model = model.to(device)\n",
    "            modelname = textType+'_'+treeType+'-tree_pretrain-'+str(pretrainType)\n",
    "            print(modelname)\n",
    "            trainModel(model,modelname)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "treeparams = {\n",
    "    'cuda': torch.cuda.is_available(),\n",
    "    'in_dim':8,\n",
    "    'mem_dim':100,\n",
    "    'userVects':userVects,\n",
    "    'labels':twitter15_labels,\n",
    "    'labelMap':labelMap,\n",
    "    'criterion':nn.CrossEntropyLoss()\n",
    "}\n",
    "\n",
    "textparams = {\n",
    "    'embedding_dim':256,\n",
    "    'hidden_dim': 50,\n",
    "    'output_dim':4,\n",
    "    'num_layers':1,\n",
    "    'bidir':True,\n",
    "    'rnnType':'gru'\n",
    "}\n",
    "\n",
    "treeTypes = ['standard']\n",
    "textTypes = ['rnn']\n",
    "pretrainTypes = [False,]\n",
    "bidirTypes = [False]\n",
    "rnnTypes = ['gru']\n",
    "\n",
    "for textType in textTypes:\n",
    "    for treeType in treeTypes:\n",
    "        for pretrainType in pretrainTypes:\n",
    "            for rnnType in rnnTypes:\n",
    "                for bidirType in bidirTypes:\n",
    "                    textparams['rnnType'] = rnnType\n",
    "                    textparams['bidirType'] = bidirType\n",
    "                    \n",
    "                    model = jointModel(treeType,textType,pretrainType,textparams,treeparams,X_text,y,device)\n",
    "                    model = model.to(device)\n",
    "                    modelname = textType+'_'+treeType+'-tree_pretrain-'+str(pretrainType)+'_'+rnnType+'bidir-'+str(bidirType)\n",
    "                    print(modelname)\n",
    "                    trainModel(model,modelname)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "fakenews",
   "language": "python",
   "name": "fakenews"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
